{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "g:\\software\\python3.6\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From g:\\software\\python3.6\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./mnist_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From g:\\software\\python3.6\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./mnist_data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From g:\\software\\python3.6\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting ./mnist_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting ./mnist_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From g:\\software\\python3.6\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = './mnist_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一个非常非常简陋的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W = tf.Variable(tf.zeros([784, 10]))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "y = tf.matmul(x, W) + b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义我们的ground truth 占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-bf86c3447efc>:11: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "然后我们运行3k个step(5 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train\n",
    "for _ in range(3000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9198\n"
     ]
    }
   ],
   "source": [
    "  # Test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.尝试添加隐层"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.1 添加一层隐层,使用relu激活函数，隐层用50个神经元，初始化用随机高斯分布，学习率用0.01，循环次数30000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:53.298702239990234\n",
      "0.1347\n",
      "the 500 setps AND the loss on the train is:5.172697067260742\n",
      "0.6219\n",
      "the 1000 setps AND the loss on the train is:2.0611932277679443\n",
      "0.6929\n",
      "the 1500 setps AND the loss on the train is:3.164088726043701\n",
      "0.7207\n",
      "the 2000 setps AND the loss on the train is:1.6347392797470093\n",
      "0.7356\n",
      "the 2500 setps AND the loss on the train is:1.6793752908706665\n",
      "0.7462\n",
      "the 3000 setps AND the loss on the train is:1.779660940170288\n",
      "0.7522\n",
      "the 3500 setps AND the loss on the train is:2.0791966915130615\n",
      "0.759\n",
      "the 4000 setps AND the loss on the train is:0.8019480109214783\n",
      "0.7665\n",
      "the 4500 setps AND the loss on the train is:1.0183169841766357\n",
      "0.7703\n",
      "the 5000 setps AND the loss on the train is:1.2982765436172485\n",
      "0.7767\n",
      "the 5500 setps AND the loss on the train is:1.1889513731002808\n",
      "0.7824\n",
      "the 6000 setps AND the loss on the train is:0.8731803297996521\n",
      "0.7847\n",
      "the 6500 setps AND the loss on the train is:0.9811608195304871\n",
      "0.7912\n",
      "the 7000 setps AND the loss on the train is:0.6163556575775146\n",
      "0.7971\n",
      "the 7500 setps AND the loss on the train is:0.6837205290794373\n",
      "0.799\n",
      "the 8000 setps AND the loss on the train is:1.052347183227539\n",
      "0.8018\n",
      "the 8500 setps AND the loss on the train is:0.4458686411380768\n",
      "0.8067\n",
      "the 9000 setps AND the loss on the train is:0.5847633481025696\n",
      "0.8104\n",
      "the 9500 setps AND the loss on the train is:0.6123715043067932\n",
      "0.8133\n",
      "the 10000 setps AND the loss on the train is:0.6710918545722961\n",
      "0.8141\n",
      "the 10500 setps AND the loss on the train is:0.599790632724762\n",
      "0.817\n",
      "the 11000 setps AND the loss on the train is:0.6413697600364685\n",
      "0.8213\n",
      "the 11500 setps AND the loss on the train is:0.6025654077529907\n",
      "0.8248\n",
      "the 12000 setps AND the loss on the train is:0.5873026847839355\n",
      "0.829\n",
      "the 12500 setps AND the loss on the train is:0.5031729340553284\n",
      "0.8314\n",
      "the 13000 setps AND the loss on the train is:0.5873755812644958\n",
      "0.8317\n",
      "the 13500 setps AND the loss on the train is:0.3208518326282501\n",
      "0.8359\n",
      "the 14000 setps AND the loss on the train is:0.7495747208595276\n",
      "0.8391\n",
      "the 14500 setps AND the loss on the train is:0.49829110503196716\n",
      "0.8421\n",
      "the 15000 setps AND the loss on the train is:0.4488510489463806\n",
      "0.8412\n",
      "the 15500 setps AND the loss on the train is:0.43350517749786377\n",
      "0.844\n",
      "the 16000 setps AND the loss on the train is:0.5597792267799377\n",
      "0.8448\n",
      "the 16500 setps AND the loss on the train is:0.4733474850654602\n",
      "0.8475\n",
      "the 17000 setps AND the loss on the train is:0.5104091167449951\n",
      "0.8485\n",
      "the 17500 setps AND the loss on the train is:0.318207323551178\n",
      "0.8495\n",
      "the 18000 setps AND the loss on the train is:0.6402313113212585\n",
      "0.8516\n",
      "the 18500 setps AND the loss on the train is:0.616146445274353\n",
      "0.852\n",
      "the 19000 setps AND the loss on the train is:0.37564483284950256\n",
      "0.8549\n",
      "the 19500 setps AND the loss on the train is:0.5265666246414185\n",
      "0.8549\n",
      "the 20000 setps AND the loss on the train is:0.4253741502761841\n",
      "0.8581\n",
      "the 20500 setps AND the loss on the train is:0.3672877252101898\n",
      "0.8585\n",
      "the 21000 setps AND the loss on the train is:0.32459190487861633\n",
      "0.8595\n",
      "the 21500 setps AND the loss on the train is:0.6157328486442566\n",
      "0.8582\n",
      "the 22000 setps AND the loss on the train is:0.42652419209480286\n",
      "0.8607\n",
      "the 22500 setps AND the loss on the train is:0.43158775568008423\n",
      "0.8609\n",
      "the 23000 setps AND the loss on the train is:0.31077954173088074\n",
      "0.8618\n",
      "the 23500 setps AND the loss on the train is:0.35205477476119995\n",
      "0.8637\n",
      "the 24000 setps AND the loss on the train is:0.5033557415008545\n",
      "0.8627\n",
      "the 24500 setps AND the loss on the train is:0.41275811195373535\n",
      "0.866\n",
      "the 25000 setps AND the loss on the train is:0.551189661026001\n",
      "0.8625\n",
      "the 25500 setps AND the loss on the train is:0.39077702164649963\n",
      "0.868\n",
      "the 26000 setps AND the loss on the train is:0.3603402376174927\n",
      "0.8673\n",
      "the 26500 setps AND the loss on the train is:0.7101263999938965\n",
      "0.8672\n",
      "the 27000 setps AND the loss on the train is:0.35547763109207153\n",
      "0.8701\n",
      "the 27500 setps AND the loss on the train is:0.31190717220306396\n",
      "0.8702\n",
      "the 28000 setps AND the loss on the train is:0.42728596925735474\n",
      "0.8717\n",
      "the 28500 setps AND the loss on the train is:0.4007957875728607\n",
      "0.8695\n",
      "the 29000 setps AND the loss on the train is:0.20122316479682922\n",
      "0.8731\n",
      "the 29500 setps AND the loss on the train is:0.2763257920742035\n",
      "0.8737\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "w1=tf.Variable(tf.random_normal([784,50]))\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=tf.Variable(tf.random_normal([50,10]))\n",
    "b2=tf.Variable(tf.random_normal([10]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits2))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits2,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))\n",
    "        \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.2 添加一层隐层,使用sigmoid激活函数，隐层用50个神经元，初始化用随机高斯分布，学习率用0.01，循环次数30000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:5.281957626342773\n",
      "0.1204\n",
      "the 500 setps AND the loss on the train is:3.036067485809326\n",
      "0.2334\n",
      "the 1000 setps AND the loss on the train is:2.3716976642608643\n",
      "0.3526\n",
      "the 1500 setps AND the loss on the train is:1.6300097703933716\n",
      "0.4352\n",
      "the 2000 setps AND the loss on the train is:1.5740858316421509\n",
      "0.4927\n",
      "the 2500 setps AND the loss on the train is:1.4724972248077393\n",
      "0.538\n",
      "the 3000 setps AND the loss on the train is:1.6322270631790161\n",
      "0.5704\n",
      "the 3500 setps AND the loss on the train is:1.3935720920562744\n",
      "0.5994\n",
      "the 4000 setps AND the loss on the train is:1.3648970127105713\n",
      "0.6227\n",
      "the 4500 setps AND the loss on the train is:1.2535536289215088\n",
      "0.6409\n",
      "the 5000 setps AND the loss on the train is:1.0474271774291992\n",
      "0.6586\n",
      "the 5500 setps AND the loss on the train is:1.0662747621536255\n",
      "0.6731\n",
      "the 6000 setps AND the loss on the train is:0.9034923315048218\n",
      "0.6849\n",
      "the 6500 setps AND the loss on the train is:0.8397930264472961\n",
      "0.6968\n",
      "the 7000 setps AND the loss on the train is:0.7933135032653809\n",
      "0.7066\n",
      "the 7500 setps AND the loss on the train is:0.6554902791976929\n",
      "0.716\n",
      "the 8000 setps AND the loss on the train is:0.9217042326927185\n",
      "0.7235\n",
      "the 8500 setps AND the loss on the train is:0.7369518280029297\n",
      "0.7327\n",
      "the 9000 setps AND the loss on the train is:0.8694265484809875\n",
      "0.7375\n",
      "the 9500 setps AND the loss on the train is:0.747745156288147\n",
      "0.7454\n",
      "the 10000 setps AND the loss on the train is:1.01358962059021\n",
      "0.7506\n",
      "the 10500 setps AND the loss on the train is:0.7549217343330383\n",
      "0.756\n",
      "the 11000 setps AND the loss on the train is:0.6286512613296509\n",
      "0.7611\n",
      "the 11500 setps AND the loss on the train is:0.9013727307319641\n",
      "0.7663\n",
      "the 12000 setps AND the loss on the train is:0.6777912974357605\n",
      "0.771\n",
      "the 12500 setps AND the loss on the train is:0.6699817776679993\n",
      "0.7747\n",
      "the 13000 setps AND the loss on the train is:0.9031868577003479\n",
      "0.7798\n",
      "the 13500 setps AND the loss on the train is:0.6727063059806824\n",
      "0.7836\n",
      "the 14000 setps AND the loss on the train is:0.6563522219657898\n",
      "0.7864\n",
      "the 14500 setps AND the loss on the train is:0.5800243616104126\n",
      "0.7899\n",
      "the 15000 setps AND the loss on the train is:0.6113001108169556\n",
      "0.7926\n",
      "the 15500 setps AND the loss on the train is:0.4902813732624054\n",
      "0.7954\n",
      "the 16000 setps AND the loss on the train is:0.5855398774147034\n",
      "0.7989\n",
      "the 16500 setps AND the loss on the train is:0.6506906151771545\n",
      "0.8001\n",
      "the 17000 setps AND the loss on the train is:0.4900096654891968\n",
      "0.8017\n",
      "the 17500 setps AND the loss on the train is:0.6019062399864197\n",
      "0.8043\n",
      "the 18000 setps AND the loss on the train is:0.8228468298912048\n",
      "0.8068\n",
      "the 18500 setps AND the loss on the train is:0.9104516506195068\n",
      "0.8086\n",
      "the 19000 setps AND the loss on the train is:0.7912828326225281\n",
      "0.8098\n",
      "the 19500 setps AND the loss on the train is:0.6420738101005554\n",
      "0.8119\n",
      "the 20000 setps AND the loss on the train is:0.6259877681732178\n",
      "0.8129\n",
      "the 20500 setps AND the loss on the train is:0.8513872027397156\n",
      "0.8152\n",
      "the 21000 setps AND the loss on the train is:0.597042977809906\n",
      "0.8166\n",
      "the 21500 setps AND the loss on the train is:0.5709565877914429\n",
      "0.8176\n",
      "the 22000 setps AND the loss on the train is:0.4558597207069397\n",
      "0.8209\n",
      "the 22500 setps AND the loss on the train is:0.42462357878685\n",
      "0.822\n",
      "the 23000 setps AND the loss on the train is:0.7264964580535889\n",
      "0.8232\n",
      "the 23500 setps AND the loss on the train is:0.5234251022338867\n",
      "0.8253\n",
      "the 24000 setps AND the loss on the train is:0.5446734428405762\n",
      "0.8261\n",
      "the 24500 setps AND the loss on the train is:0.8567192554473877\n",
      "0.8276\n",
      "the 25000 setps AND the loss on the train is:0.6108092069625854\n",
      "0.8298\n",
      "the 25500 setps AND the loss on the train is:0.6172856688499451\n",
      "0.831\n",
      "the 26000 setps AND the loss on the train is:0.5555438995361328\n",
      "0.8323\n",
      "the 26500 setps AND the loss on the train is:0.5994518399238586\n",
      "0.8329\n",
      "the 27000 setps AND the loss on the train is:0.571031928062439\n",
      "0.8333\n",
      "the 27500 setps AND the loss on the train is:0.39151039719581604\n",
      "0.8358\n",
      "the 28000 setps AND the loss on the train is:0.5960670709609985\n",
      "0.8365\n",
      "the 28500 setps AND the loss on the train is:0.7173928618431091\n",
      "0.838\n",
      "the 29000 setps AND the loss on the train is:0.7217162251472473\n",
      "0.8384\n",
      "the 29500 setps AND the loss on the train is:0.7315689325332642\n",
      "0.8401\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "w1=tf.Variable(tf.random_normal([784,50]))\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.sigmoid(logits1)\n",
    "\n",
    "w2=tf.Variable(tf.random_normal([50,10]))\n",
    "b2=tf.Variable(tf.random_normal([10]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits2))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits2,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.3 添加一层隐层,使用relu激活函数，隐层用500个神经元，初始化用随机高斯分布，学习率用0.01，循环次数30000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:184.40560913085938\n",
      "0.104\n",
      "the 500 setps AND the loss on the train is:11.617074966430664\n",
      "0.8248\n",
      "the 1000 setps AND the loss on the train is:8.477649688720703\n",
      "0.8574\n",
      "the 1500 setps AND the loss on the train is:5.372708320617676\n",
      "0.8749\n",
      "the 2000 setps AND the loss on the train is:5.8146443367004395\n",
      "0.8833\n",
      "the 2500 setps AND the loss on the train is:3.5455591678619385\n",
      "0.8903\n",
      "the 3000 setps AND the loss on the train is:7.078518867492676\n",
      "0.8923\n",
      "the 3500 setps AND the loss on the train is:1.5772556066513062\n",
      "0.8978\n",
      "the 4000 setps AND the loss on the train is:2.535456657409668\n",
      "0.9033\n",
      "the 4500 setps AND the loss on the train is:4.038296699523926\n",
      "0.9056\n",
      "the 5000 setps AND the loss on the train is:2.311605453491211\n",
      "0.9057\n",
      "the 5500 setps AND the loss on the train is:2.9654550552368164\n",
      "0.9082\n",
      "the 6000 setps AND the loss on the train is:1.1977192163467407\n",
      "0.9096\n",
      "the 6500 setps AND the loss on the train is:3.2771716117858887\n",
      "0.9129\n",
      "the 7000 setps AND the loss on the train is:2.060824394226074\n",
      "0.9152\n",
      "the 7500 setps AND the loss on the train is:2.4196059703826904\n",
      "0.9155\n",
      "the 8000 setps AND the loss on the train is:1.1840224266052246\n",
      "0.9175\n",
      "the 8500 setps AND the loss on the train is:2.528191328048706\n",
      "0.9184\n",
      "the 9000 setps AND the loss on the train is:2.760809898376465\n",
      "0.9197\n",
      "the 9500 setps AND the loss on the train is:0.003899066476151347\n",
      "0.9207\n",
      "the 10000 setps AND the loss on the train is:0.9932727217674255\n",
      "0.9219\n",
      "the 10500 setps AND the loss on the train is:0.7502629160881042\n",
      "0.9214\n",
      "the 11000 setps AND the loss on the train is:1.4795751571655273\n",
      "0.9218\n",
      "the 11500 setps AND the loss on the train is:0.16050732135772705\n",
      "0.9218\n",
      "the 12000 setps AND the loss on the train is:5.494189262390137\n",
      "0.9245\n",
      "the 12500 setps AND the loss on the train is:1.8042434453964233\n",
      "0.9236\n",
      "the 13000 setps AND the loss on the train is:1.9688552618026733\n",
      "0.9247\n",
      "the 13500 setps AND the loss on the train is:4.784000396728516\n",
      "0.9248\n",
      "the 14000 setps AND the loss on the train is:2.2043418884277344\n",
      "0.9238\n",
      "the 14500 setps AND the loss on the train is:0.6010172367095947\n",
      "0.9251\n",
      "the 15000 setps AND the loss on the train is:0.5137166380882263\n",
      "0.9261\n",
      "the 15500 setps AND the loss on the train is:0.02857821248471737\n",
      "0.9281\n",
      "the 16000 setps AND the loss on the train is:0.5795416831970215\n",
      "0.9275\n",
      "the 16500 setps AND the loss on the train is:0.41248223185539246\n",
      "0.9278\n",
      "the 17000 setps AND the loss on the train is:0.5232260227203369\n",
      "0.9289\n",
      "the 17500 setps AND the loss on the train is:1.8861314058303833\n",
      "0.9296\n",
      "the 18000 setps AND the loss on the train is:0.2763119339942932\n",
      "0.9297\n",
      "the 18500 setps AND the loss on the train is:0.7060576677322388\n",
      "0.9295\n",
      "the 19000 setps AND the loss on the train is:0.9513120055198669\n",
      "0.9302\n",
      "the 19500 setps AND the loss on the train is:1.9986644983291626\n",
      "0.9297\n",
      "the 20000 setps AND the loss on the train is:1.8925433158874512\n",
      "0.9302\n",
      "the 20500 setps AND the loss on the train is:1.1794732809066772\n",
      "0.9291\n",
      "the 21000 setps AND the loss on the train is:0.03593948483467102\n",
      "0.9316\n",
      "the 21500 setps AND the loss on the train is:0.26923272013664246\n",
      "0.932\n",
      "the 22000 setps AND the loss on the train is:0.3931083381175995\n",
      "0.9307\n",
      "the 22500 setps AND the loss on the train is:0.37050822377204895\n",
      "0.9324\n",
      "the 23000 setps AND the loss on the train is:0.7096585035324097\n",
      "0.9311\n",
      "the 23500 setps AND the loss on the train is:0.002302108332514763\n",
      "0.9317\n",
      "the 24000 setps AND the loss on the train is:0.0009119430324062705\n",
      "0.933\n",
      "the 24500 setps AND the loss on the train is:0.39859870076179504\n",
      "0.933\n",
      "the 25000 setps AND the loss on the train is:0.0005174450343474746\n",
      "0.9327\n",
      "the 25500 setps AND the loss on the train is:0.0005901705008000135\n",
      "0.9336\n",
      "the 26000 setps AND the loss on the train is:0.042893122881650925\n",
      "0.9328\n",
      "the 26500 setps AND the loss on the train is:0.15969765186309814\n",
      "0.9344\n",
      "the 27000 setps AND the loss on the train is:0.3391232192516327\n",
      "0.9333\n",
      "the 27500 setps AND the loss on the train is:0.967439591884613\n",
      "0.9341\n",
      "the 28000 setps AND the loss on the train is:0.6239391565322876\n",
      "0.9344\n",
      "the 28500 setps AND the loss on the train is:0.2544649839401245\n",
      "0.9336\n",
      "the 29000 setps AND the loss on the train is:0.2499028444290161\n",
      "0.9346\n",
      "the 29500 setps AND the loss on the train is:0.49060916900634766\n",
      "0.9347\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "w1=tf.Variable(tf.random_normal([784,500]))\n",
    "b1=tf.Variable(tf.random_normal([500]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=tf.Variable(tf.random_normal([500,10]))\n",
    "b2=tf.Variable(tf.random_normal([10]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits2))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits2,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.4 添加一层隐层,使用relu激活函数，隐层用50个神经元，初始化用随机高斯分布，学习率用0.001，循环次数30000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:103.50450897216797\n",
      "0.0961\n",
      "the 500 setps AND the loss on the train is:25.15308380126953\n",
      "0.2684\n",
      "the 1000 setps AND the loss on the train is:17.23932456970215\n",
      "0.3928\n",
      "the 1500 setps AND the loss on the train is:11.44575309753418\n",
      "0.4727\n",
      "the 2000 setps AND the loss on the train is:8.937634468078613\n",
      "0.5248\n",
      "the 2500 setps AND the loss on the train is:8.335046768188477\n",
      "0.5594\n",
      "the 3000 setps AND the loss on the train is:6.612361431121826\n",
      "0.588\n",
      "the 3500 setps AND the loss on the train is:6.519354820251465\n",
      "0.6125\n",
      "the 4000 setps AND the loss on the train is:6.7640700340271\n",
      "0.6314\n",
      "the 4500 setps AND the loss on the train is:5.464339733123779\n",
      "0.6449\n",
      "the 5000 setps AND the loss on the train is:3.5313658714294434\n",
      "0.6575\n",
      "the 5500 setps AND the loss on the train is:6.088150501251221\n",
      "0.6689\n",
      "the 6000 setps AND the loss on the train is:5.685321807861328\n",
      "0.6764\n",
      "the 6500 setps AND the loss on the train is:6.069891929626465\n",
      "0.6848\n",
      "the 7000 setps AND the loss on the train is:3.77321195602417\n",
      "0.6921\n",
      "the 7500 setps AND the loss on the train is:4.007122993469238\n",
      "0.6998\n",
      "the 8000 setps AND the loss on the train is:6.3004302978515625\n",
      "0.7062\n",
      "the 8500 setps AND the loss on the train is:3.287838935852051\n",
      "0.7115\n",
      "the 9000 setps AND the loss on the train is:4.265174388885498\n",
      "0.7164\n",
      "the 9500 setps AND the loss on the train is:2.6467161178588867\n",
      "0.7223\n",
      "the 10000 setps AND the loss on the train is:3.4825174808502197\n",
      "0.7262\n",
      "the 10500 setps AND the loss on the train is:2.7412078380584717\n",
      "0.7309\n",
      "the 11000 setps AND the loss on the train is:2.657960891723633\n",
      "0.7357\n",
      "the 11500 setps AND the loss on the train is:2.9591314792633057\n",
      "0.7393\n",
      "the 12000 setps AND the loss on the train is:2.376469612121582\n",
      "0.742\n",
      "the 12500 setps AND the loss on the train is:3.120400905609131\n",
      "0.7454\n",
      "the 13000 setps AND the loss on the train is:2.716179847717285\n",
      "0.7486\n",
      "the 13500 setps AND the loss on the train is:2.8266546726226807\n",
      "0.7512\n",
      "the 14000 setps AND the loss on the train is:2.7740848064422607\n",
      "0.7527\n",
      "the 14500 setps AND the loss on the train is:2.111811399459839\n",
      "0.7553\n",
      "the 15000 setps AND the loss on the train is:2.446730852127075\n",
      "0.7577\n",
      "the 15500 setps AND the loss on the train is:2.156852960586548\n",
      "0.7597\n",
      "the 16000 setps AND the loss on the train is:1.5590568780899048\n",
      "0.7609\n",
      "the 16500 setps AND the loss on the train is:1.4508073329925537\n",
      "0.7632\n",
      "the 17000 setps AND the loss on the train is:1.2774864435195923\n",
      "0.765\n",
      "the 17500 setps AND the loss on the train is:1.9992221593856812\n",
      "0.7667\n",
      "the 18000 setps AND the loss on the train is:1.7431029081344604\n",
      "0.7679\n",
      "the 18500 setps AND the loss on the train is:2.0560507774353027\n",
      "0.7686\n",
      "the 19000 setps AND the loss on the train is:1.5015069246292114\n",
      "0.7694\n",
      "the 19500 setps AND the loss on the train is:1.8737821578979492\n",
      "0.7706\n",
      "the 20000 setps AND the loss on the train is:1.8128559589385986\n",
      "0.7703\n",
      "the 20500 setps AND the loss on the train is:0.9491903185844421\n",
      "0.7721\n",
      "the 21000 setps AND the loss on the train is:1.6303534507751465\n",
      "0.7735\n",
      "the 21500 setps AND the loss on the train is:2.0983223915100098\n",
      "0.7748\n",
      "the 22000 setps AND the loss on the train is:1.765607476234436\n",
      "0.7751\n",
      "the 22500 setps AND the loss on the train is:2.763563871383667\n",
      "0.7763\n",
      "the 23000 setps AND the loss on the train is:2.3322410583496094\n",
      "0.7776\n",
      "the 23500 setps AND the loss on the train is:1.5753077268600464\n",
      "0.777\n",
      "the 24000 setps AND the loss on the train is:1.397433876991272\n",
      "0.7791\n",
      "the 24500 setps AND the loss on the train is:2.561386823654175\n",
      "0.7791\n",
      "the 25000 setps AND the loss on the train is:2.3432576656341553\n",
      "0.7798\n",
      "the 25500 setps AND the loss on the train is:1.716280460357666\n",
      "0.781\n",
      "the 26000 setps AND the loss on the train is:1.7466293573379517\n",
      "0.7818\n",
      "the 26500 setps AND the loss on the train is:1.1326839923858643\n",
      "0.7815\n",
      "the 27000 setps AND the loss on the train is:1.4248865842819214\n",
      "0.7838\n",
      "the 27500 setps AND the loss on the train is:0.9588416218757629\n",
      "0.7841\n",
      "the 28000 setps AND the loss on the train is:1.3745898008346558\n",
      "0.7838\n",
      "the 28500 setps AND the loss on the train is:1.7334644794464111\n",
      "0.7865\n",
      "the 29000 setps AND the loss on the train is:0.755068302154541\n",
      "0.7871\n",
      "the 29500 setps AND the loss on the train is:1.2592283487319946\n",
      "0.7876\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "w1=tf.Variable(tf.random_normal([784,50]))\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=tf.Variable(tf.random_normal([50,10]))\n",
    "b2=tf.Variable(tf.random_normal([10]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits2))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.001).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits2,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.5 添加一层隐层,使用relu激活函数，隐层用50个神经元，初始化用随机高斯分布，学习率用0.01，循环次数60000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:67.66009521484375\n",
      "0.0995\n",
      "the 500 setps AND the loss on the train is:7.890432357788086\n",
      "0.6505\n",
      "the 1000 setps AND the loss on the train is:2.8513758182525635\n",
      "0.7214\n",
      "the 1500 setps AND the loss on the train is:1.8352011442184448\n",
      "0.7425\n",
      "the 2000 setps AND the loss on the train is:2.7324182987213135\n",
      "0.7581\n",
      "the 2500 setps AND the loss on the train is:1.6717557907104492\n",
      "0.7683\n",
      "the 3000 setps AND the loss on the train is:1.271074891090393\n",
      "0.7736\n",
      "the 3500 setps AND the loss on the train is:0.8559507131576538\n",
      "0.7815\n",
      "the 4000 setps AND the loss on the train is:1.1328246593475342\n",
      "0.7853\n",
      "the 4500 setps AND the loss on the train is:0.7882671356201172\n",
      "0.7887\n",
      "the 5000 setps AND the loss on the train is:0.7236754894256592\n",
      "0.7941\n",
      "the 5500 setps AND the loss on the train is:0.5020145177841187\n",
      "0.7935\n",
      "the 6000 setps AND the loss on the train is:0.7217186689376831\n",
      "0.8051\n",
      "the 6500 setps AND the loss on the train is:0.8300124406814575\n",
      "0.8043\n",
      "the 7000 setps AND the loss on the train is:0.6509562730789185\n",
      "0.8061\n",
      "the 7500 setps AND the loss on the train is:0.5427399277687073\n",
      "0.8121\n",
      "the 8000 setps AND the loss on the train is:0.5773572325706482\n",
      "0.8128\n",
      "the 8500 setps AND the loss on the train is:0.6594501733779907\n",
      "0.8167\n",
      "the 9000 setps AND the loss on the train is:0.7575319409370422\n",
      "0.8201\n",
      "the 9500 setps AND the loss on the train is:0.32409125566482544\n",
      "0.8235\n",
      "the 10000 setps AND the loss on the train is:0.5135864019393921\n",
      "0.8232\n",
      "the 10500 setps AND the loss on the train is:0.5171781182289124\n",
      "0.8266\n",
      "the 11000 setps AND the loss on the train is:0.34278857707977295\n",
      "0.8235\n",
      "the 11500 setps AND the loss on the train is:0.4291507601737976\n",
      "0.8316\n",
      "the 12000 setps AND the loss on the train is:0.4680473208427429\n",
      "0.8288\n",
      "the 12500 setps AND the loss on the train is:0.47320830821990967\n",
      "0.832\n",
      "the 13000 setps AND the loss on the train is:0.3886333405971527\n",
      "0.8299\n",
      "the 13500 setps AND the loss on the train is:0.6018779277801514\n",
      "0.8375\n",
      "the 14000 setps AND the loss on the train is:0.4756205081939697\n",
      "0.8392\n",
      "the 14500 setps AND the loss on the train is:0.3636611998081207\n",
      "0.8364\n",
      "the 15000 setps AND the loss on the train is:0.5317489504814148\n",
      "0.841\n",
      "the 15500 setps AND the loss on the train is:0.6168285608291626\n",
      "0.8432\n",
      "the 16000 setps AND the loss on the train is:0.4763527810573578\n",
      "0.8456\n",
      "the 16500 setps AND the loss on the train is:0.70484459400177\n",
      "0.8453\n",
      "the 17000 setps AND the loss on the train is:0.39387091994285583\n",
      "0.8444\n",
      "the 17500 setps AND the loss on the train is:0.5062103271484375\n",
      "0.8483\n",
      "the 18000 setps AND the loss on the train is:0.33344578742980957\n",
      "0.8514\n",
      "the 18500 setps AND the loss on the train is:0.4009474217891693\n",
      "0.8512\n",
      "the 19000 setps AND the loss on the train is:0.2708272635936737\n",
      "0.8539\n",
      "the 19500 setps AND the loss on the train is:0.5426192879676819\n",
      "0.8527\n",
      "the 20000 setps AND the loss on the train is:0.44621744751930237\n",
      "0.8532\n",
      "the 20500 setps AND the loss on the train is:0.528937578201294\n",
      "0.8535\n",
      "the 21000 setps AND the loss on the train is:0.4305518865585327\n",
      "0.8589\n",
      "the 21500 setps AND the loss on the train is:0.5127288699150085\n",
      "0.8549\n",
      "the 22000 setps AND the loss on the train is:0.2961026132106781\n",
      "0.8605\n",
      "the 22500 setps AND the loss on the train is:0.9030041694641113\n",
      "0.8602\n",
      "the 23000 setps AND the loss on the train is:0.5027036070823669\n",
      "0.8598\n",
      "the 23500 setps AND the loss on the train is:0.5349389910697937\n",
      "0.862\n",
      "the 24000 setps AND the loss on the train is:0.362039715051651\n",
      "0.8626\n",
      "the 24500 setps AND the loss on the train is:0.4281211793422699\n",
      "0.8624\n",
      "the 25000 setps AND the loss on the train is:0.3280807137489319\n",
      "0.8623\n",
      "the 25500 setps AND the loss on the train is:0.3871743083000183\n",
      "0.8676\n",
      "the 26000 setps AND the loss on the train is:0.5644422769546509\n",
      "0.8636\n",
      "the 26500 setps AND the loss on the train is:0.4131311774253845\n",
      "0.8656\n",
      "the 27000 setps AND the loss on the train is:0.43755194544792175\n",
      "0.8689\n",
      "the 27500 setps AND the loss on the train is:0.3913605511188507\n",
      "0.8636\n",
      "the 28000 setps AND the loss on the train is:0.23254121840000153\n",
      "0.8688\n",
      "the 28500 setps AND the loss on the train is:0.3982986509799957\n",
      "0.8669\n",
      "the 29000 setps AND the loss on the train is:0.30429789423942566\n",
      "0.8698\n",
      "the 29500 setps AND the loss on the train is:0.3756904602050781\n",
      "0.8751\n",
      "the 30000 setps AND the loss on the train is:0.32026368379592896\n",
      "0.8708\n",
      "the 30500 setps AND the loss on the train is:0.3361109793186188\n",
      "0.87\n",
      "the 31000 setps AND the loss on the train is:0.38568034768104553\n",
      "0.8741\n",
      "the 31500 setps AND the loss on the train is:0.19110725820064545\n",
      "0.8726\n",
      "the 32000 setps AND the loss on the train is:0.333099365234375\n",
      "0.8743\n",
      "the 32500 setps AND the loss on the train is:0.27904900908470154\n",
      "0.8752\n",
      "the 33000 setps AND the loss on the train is:0.30756425857543945\n",
      "0.8729\n",
      "the 33500 setps AND the loss on the train is:0.4604949653148651\n",
      "0.8721\n",
      "the 34000 setps AND the loss on the train is:0.3590191602706909\n",
      "0.8762\n",
      "the 34500 setps AND the loss on the train is:0.45067736506462097\n",
      "0.8757\n",
      "the 35000 setps AND the loss on the train is:0.23337990045547485\n",
      "0.8778\n",
      "the 35500 setps AND the loss on the train is:0.3187490701675415\n",
      "0.88\n",
      "the 36000 setps AND the loss on the train is:0.19705815613269806\n",
      "0.8769\n",
      "the 36500 setps AND the loss on the train is:0.3151454031467438\n",
      "0.8781\n",
      "the 37000 setps AND the loss on the train is:0.6024571657180786\n",
      "0.8758\n",
      "the 37500 setps AND the loss on the train is:0.32184505462646484\n",
      "0.8789\n",
      "the 38000 setps AND the loss on the train is:0.5171888470649719\n",
      "0.8784\n",
      "the 38500 setps AND the loss on the train is:0.386397123336792\n",
      "0.8822\n",
      "the 39000 setps AND the loss on the train is:0.38058117032051086\n",
      "0.882\n",
      "the 39500 setps AND the loss on the train is:0.27452003955841064\n",
      "0.8839\n",
      "the 40000 setps AND the loss on the train is:0.32690688967704773\n",
      "0.8831\n",
      "the 40500 setps AND the loss on the train is:0.33567073941230774\n",
      "0.882\n",
      "the 41000 setps AND the loss on the train is:0.5391890406608582\n",
      "0.8829\n",
      "the 41500 setps AND the loss on the train is:0.22854973375797272\n",
      "0.8835\n",
      "the 42000 setps AND the loss on the train is:0.3937622308731079\n",
      "0.8798\n",
      "the 42500 setps AND the loss on the train is:0.33722779154777527\n",
      "0.884\n",
      "the 43000 setps AND the loss on the train is:0.2899788022041321\n",
      "0.8821\n",
      "the 43500 setps AND the loss on the train is:0.277937650680542\n",
      "0.8842\n",
      "the 44000 setps AND the loss on the train is:0.4054332375526428\n",
      "0.8856\n",
      "the 44500 setps AND the loss on the train is:0.4914751350879669\n",
      "0.8862\n",
      "the 45000 setps AND the loss on the train is:0.3232435882091522\n",
      "0.8858\n",
      "the 45500 setps AND the loss on the train is:0.8330522775650024\n",
      "0.8876\n",
      "the 46000 setps AND the loss on the train is:0.3503032624721527\n",
      "0.8844\n",
      "the 46500 setps AND the loss on the train is:0.34287330508232117\n",
      "0.8875\n",
      "the 47000 setps AND the loss on the train is:0.2628841698169708\n",
      "0.8861\n",
      "the 47500 setps AND the loss on the train is:0.4334810674190521\n",
      "0.8888\n",
      "the 48000 setps AND the loss on the train is:0.2127719521522522\n",
      "0.8887\n",
      "the 48500 setps AND the loss on the train is:0.4031623601913452\n",
      "0.8883\n",
      "the 49000 setps AND the loss on the train is:0.46201038360595703\n",
      "0.8885\n",
      "the 49500 setps AND the loss on the train is:0.31745055317878723\n",
      "0.8898\n",
      "the 50000 setps AND the loss on the train is:0.23570699989795685\n",
      "0.891\n",
      "the 50500 setps AND the loss on the train is:0.25929394364356995\n",
      "0.8902\n",
      "the 51000 setps AND the loss on the train is:0.3558366894721985\n",
      "0.8905\n",
      "the 51500 setps AND the loss on the train is:0.2866186201572418\n",
      "0.8904\n",
      "the 52000 setps AND the loss on the train is:0.3989112973213196\n",
      "0.8895\n",
      "the 52500 setps AND the loss on the train is:0.33147820830345154\n",
      "0.89\n",
      "the 53000 setps AND the loss on the train is:0.2982563376426697\n",
      "0.8892\n",
      "the 53500 setps AND the loss on the train is:0.1591818928718567\n",
      "0.8894\n",
      "the 54000 setps AND the loss on the train is:0.34681427478790283\n",
      "0.8934\n",
      "the 54500 setps AND the loss on the train is:0.2510983347892761\n",
      "0.8934\n",
      "the 55000 setps AND the loss on the train is:0.37647461891174316\n",
      "0.8934\n",
      "the 55500 setps AND the loss on the train is:0.32150667905807495\n",
      "0.8921\n",
      "the 56000 setps AND the loss on the train is:0.4087025821208954\n",
      "0.8934\n",
      "the 56500 setps AND the loss on the train is:0.28158804774284363\n",
      "0.8891\n",
      "the 57000 setps AND the loss on the train is:0.42000851035118103\n",
      "0.8903\n",
      "the 57500 setps AND the loss on the train is:0.3173472285270691\n",
      "0.895\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 58000 setps AND the loss on the train is:0.2358427792787552\n",
      "0.893\n",
      "the 58500 setps AND the loss on the train is:0.24818208813667297\n",
      "0.8952\n",
      "the 59000 setps AND the loss on the train is:0.42511165142059326\n",
      "0.8916\n",
      "the 59500 setps AND the loss on the train is:0.2887212932109833\n",
      "0.8962\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "w1=tf.Variable(tf.random_normal([784,50]))\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=tf.Variable(tf.random_normal([50,10]))\n",
    "b2=tf.Variable(tf.random_normal([10]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits2))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits2,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(60000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.添加2个3个隐层,两个隐层都用relu/50cells/0.01/30000这些参数，只是层数改变"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.1 添加2个隐层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:227.89556884765625\n",
      "0.0888\n",
      "the 500 setps AND the loss on the train is:4.019355773925781\n",
      "0.7116\n",
      "the 1000 setps AND the loss on the train is:2.3696255683898926\n",
      "0.6937\n",
      "the 1500 setps AND the loss on the train is:0.898339033126831\n",
      "0.7034\n",
      "the 2000 setps AND the loss on the train is:0.969760537147522\n",
      "0.713\n",
      "the 2500 setps AND the loss on the train is:1.142604947090149\n",
      "0.7474\n",
      "the 3000 setps AND the loss on the train is:0.8152735829353333\n",
      "0.7591\n",
      "the 3500 setps AND the loss on the train is:0.5220593810081482\n",
      "0.7577\n",
      "the 4000 setps AND the loss on the train is:0.5690504908561707\n",
      "0.7665\n",
      "the 4500 setps AND the loss on the train is:0.611452043056488\n",
      "0.7762\n",
      "the 5000 setps AND the loss on the train is:0.5184317231178284\n",
      "0.7878\n",
      "the 5500 setps AND the loss on the train is:0.7394071817398071\n",
      "0.7616\n",
      "the 6000 setps AND the loss on the train is:0.7726249098777771\n",
      "0.7943\n",
      "the 6500 setps AND the loss on the train is:0.46420422196388245\n",
      "0.7889\n",
      "the 7000 setps AND the loss on the train is:0.501865029335022\n",
      "0.8069\n",
      "the 7500 setps AND the loss on the train is:0.658087968826294\n",
      "0.8111\n",
      "the 8000 setps AND the loss on the train is:0.8104456067085266\n",
      "0.8111\n",
      "the 8500 setps AND the loss on the train is:0.4923228323459625\n",
      "0.8151\n",
      "the 9000 setps AND the loss on the train is:0.6221415400505066\n",
      "0.8181\n",
      "the 9500 setps AND the loss on the train is:0.5273770093917847\n",
      "0.8137\n",
      "the 10000 setps AND the loss on the train is:0.5475628972053528\n",
      "0.8177\n",
      "the 10500 setps AND the loss on the train is:0.29157859086990356\n",
      "0.8146\n",
      "the 11000 setps AND the loss on the train is:0.3131487965583801\n",
      "0.8269\n",
      "the 11500 setps AND the loss on the train is:0.5663759112358093\n",
      "0.8151\n",
      "the 12000 setps AND the loss on the train is:0.5091866850852966\n",
      "0.8377\n",
      "the 12500 setps AND the loss on the train is:0.3877500891685486\n",
      "0.8235\n",
      "the 13000 setps AND the loss on the train is:0.5079569816589355\n",
      "0.8378\n",
      "the 13500 setps AND the loss on the train is:0.31213754415512085\n",
      "0.8378\n",
      "the 14000 setps AND the loss on the train is:0.339644193649292\n",
      "0.8401\n",
      "the 14500 setps AND the loss on the train is:0.5830880999565125\n",
      "0.8404\n",
      "the 15000 setps AND the loss on the train is:0.3493514358997345\n",
      "0.8192\n",
      "the 15500 setps AND the loss on the train is:0.4474870264530182\n",
      "0.8411\n",
      "the 16000 setps AND the loss on the train is:0.4299291670322418\n",
      "0.8439\n",
      "the 16500 setps AND the loss on the train is:0.23058098554611206\n",
      "0.8432\n",
      "the 17000 setps AND the loss on the train is:0.32841256260871887\n",
      "0.8465\n",
      "the 17500 setps AND the loss on the train is:0.32039889693260193\n",
      "0.8472\n",
      "the 18000 setps AND the loss on the train is:0.25396931171417236\n",
      "0.8318\n",
      "the 18500 setps AND the loss on the train is:0.3360205590724945\n",
      "0.8423\n",
      "the 19000 setps AND the loss on the train is:0.3059331476688385\n",
      "0.8584\n",
      "the 19500 setps AND the loss on the train is:0.3906981647014618\n",
      "0.8534\n",
      "the 20000 setps AND the loss on the train is:0.4264233410358429\n",
      "0.8299\n",
      "the 20500 setps AND the loss on the train is:0.26939189434051514\n",
      "0.8556\n",
      "the 21000 setps AND the loss on the train is:0.27618682384490967\n",
      "0.864\n",
      "the 21500 setps AND the loss on the train is:0.42417821288108826\n",
      "0.856\n",
      "the 22000 setps AND the loss on the train is:0.589057445526123\n",
      "0.863\n",
      "the 22500 setps AND the loss on the train is:0.3441000282764435\n",
      "0.8522\n",
      "the 23000 setps AND the loss on the train is:0.27988743782043457\n",
      "0.8663\n",
      "the 23500 setps AND the loss on the train is:0.4958024322986603\n",
      "0.8619\n",
      "the 24000 setps AND the loss on the train is:0.27209389209747314\n",
      "0.8563\n",
      "the 24500 setps AND the loss on the train is:0.3841072916984558\n",
      "0.856\n",
      "the 25000 setps AND the loss on the train is:0.4056774377822876\n",
      "0.8599\n",
      "the 25500 setps AND the loss on the train is:0.3805951178073883\n",
      "0.8687\n",
      "the 26000 setps AND the loss on the train is:0.4268445074558258\n",
      "0.8684\n",
      "the 26500 setps AND the loss on the train is:0.45572134852409363\n",
      "0.8592\n",
      "the 27000 setps AND the loss on the train is:0.4418584704399109\n",
      "0.859\n",
      "the 27500 setps AND the loss on the train is:0.2504085302352905\n",
      "0.874\n",
      "the 28000 setps AND the loss on the train is:0.26595133543014526\n",
      "0.8701\n",
      "the 28500 setps AND the loss on the train is:0.44281110167503357\n",
      "0.8744\n",
      "the 29000 setps AND the loss on the train is:0.39933231472969055\n",
      "0.8707\n",
      "the 29500 setps AND the loss on the train is:0.21188224852085114\n",
      "0.8664\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "w1=tf.Variable(tf.random_normal([784,50]))\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=tf.Variable(tf.random_normal([50,50]))\n",
    "b2=tf.Variable(tf.random_normal([50]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.relu(logits2)\n",
    "\n",
    "w3=tf.Variable(tf.random_normal([50,10]))\n",
    "b3=tf.Variable(tf.random_normal([10]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits3))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits3,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.2 添加3个隐层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:773.3493041992188\n",
      "0.1318\n",
      "the 500 setps AND the loss on the train is:2.072378635406494\n",
      "0.1749\n",
      "the 1000 setps AND the loss on the train is:2.217249870300293\n",
      "0.2255\n",
      "the 1500 setps AND the loss on the train is:2.020669460296631\n",
      "0.2632\n",
      "the 2000 setps AND the loss on the train is:1.6930989027023315\n",
      "0.2963\n",
      "the 2500 setps AND the loss on the train is:1.846856951713562\n",
      "0.3062\n",
      "the 3000 setps AND the loss on the train is:1.7219438552856445\n",
      "0.3529\n",
      "the 3500 setps AND the loss on the train is:1.5204559564590454\n",
      "0.3855\n",
      "the 4000 setps AND the loss on the train is:1.9459158182144165\n",
      "0.3722\n",
      "the 4500 setps AND the loss on the train is:1.5213568210601807\n",
      "0.3874\n",
      "the 5000 setps AND the loss on the train is:1.5764648914337158\n",
      "0.4061\n",
      "the 5500 setps AND the loss on the train is:1.5892398357391357\n",
      "0.3734\n",
      "the 6000 setps AND the loss on the train is:1.6354382038116455\n",
      "0.4123\n",
      "the 6500 setps AND the loss on the train is:1.4778393507003784\n",
      "0.4222\n",
      "the 7000 setps AND the loss on the train is:1.3275398015975952\n",
      "0.4339\n",
      "the 7500 setps AND the loss on the train is:1.2799906730651855\n",
      "0.4214\n",
      "the 8000 setps AND the loss on the train is:1.4862197637557983\n",
      "0.4468\n",
      "the 8500 setps AND the loss on the train is:1.2525120973587036\n",
      "0.452\n",
      "the 9000 setps AND the loss on the train is:1.381996750831604\n",
      "0.4556\n",
      "the 9500 setps AND the loss on the train is:1.3579069375991821\n",
      "0.4868\n",
      "the 10000 setps AND the loss on the train is:1.4529523849487305\n",
      "0.4667\n",
      "the 10500 setps AND the loss on the train is:1.2082078456878662\n",
      "0.4729\n",
      "the 11000 setps AND the loss on the train is:1.2261070013046265\n",
      "0.4901\n",
      "the 11500 setps AND the loss on the train is:1.6010268926620483\n",
      "0.4747\n",
      "the 12000 setps AND the loss on the train is:1.1426124572753906\n",
      "0.5274\n",
      "the 12500 setps AND the loss on the train is:1.3477171659469604\n",
      "0.5404\n",
      "the 13000 setps AND the loss on the train is:1.2821629047393799\n",
      "0.5253\n",
      "the 13500 setps AND the loss on the train is:1.1539682149887085\n",
      "0.5403\n",
      "the 14000 setps AND the loss on the train is:1.161489725112915\n",
      "0.5323\n",
      "the 14500 setps AND the loss on the train is:1.1299002170562744\n",
      "0.5615\n",
      "the 15000 setps AND the loss on the train is:1.003726601600647\n",
      "0.5961\n",
      "the 15500 setps AND the loss on the train is:1.1232881546020508\n",
      "0.5857\n",
      "the 16000 setps AND the loss on the train is:0.9566056132316589\n",
      "0.6058\n",
      "the 16500 setps AND the loss on the train is:0.9934201836585999\n",
      "0.5973\n",
      "the 17000 setps AND the loss on the train is:0.963549017906189\n",
      "0.6306\n",
      "the 17500 setps AND the loss on the train is:0.8687063455581665\n",
      "0.6404\n",
      "the 18000 setps AND the loss on the train is:0.9035987854003906\n",
      "0.6209\n",
      "the 18500 setps AND the loss on the train is:1.0352002382278442\n",
      "0.6254\n",
      "the 19000 setps AND the loss on the train is:0.8349936008453369\n",
      "0.6507\n",
      "the 19500 setps AND the loss on the train is:0.9317992329597473\n",
      "0.6685\n",
      "the 20000 setps AND the loss on the train is:0.9221717715263367\n",
      "0.6553\n",
      "the 20500 setps AND the loss on the train is:0.961245596408844\n",
      "0.6636\n",
      "the 21000 setps AND the loss on the train is:0.8723272085189819\n",
      "0.6699\n",
      "the 21500 setps AND the loss on the train is:1.0012078285217285\n",
      "0.5988\n",
      "the 22000 setps AND the loss on the train is:1.0093151330947876\n",
      "0.6801\n",
      "the 22500 setps AND the loss on the train is:0.8738327622413635\n",
      "0.6881\n",
      "the 23000 setps AND the loss on the train is:0.7925599813461304\n",
      "0.6459\n",
      "the 23500 setps AND the loss on the train is:0.9458174109458923\n",
      "0.689\n",
      "the 24000 setps AND the loss on the train is:0.9863798022270203\n",
      "0.6849\n",
      "the 24500 setps AND the loss on the train is:0.8477866649627686\n",
      "0.6799\n",
      "the 25000 setps AND the loss on the train is:0.8378235101699829\n",
      "0.6847\n",
      "the 25500 setps AND the loss on the train is:0.8626790046691895\n",
      "0.6759\n",
      "the 26000 setps AND the loss on the train is:0.7279202342033386\n",
      "0.6982\n",
      "the 26500 setps AND the loss on the train is:0.7886687517166138\n",
      "0.7167\n",
      "the 27000 setps AND the loss on the train is:0.9372850656509399\n",
      "0.7017\n",
      "the 27500 setps AND the loss on the train is:0.9298866391181946\n",
      "0.7227\n",
      "the 28000 setps AND the loss on the train is:0.8065767884254456\n",
      "0.7051\n",
      "the 28500 setps AND the loss on the train is:0.7135530114173889\n",
      "0.728\n",
      "the 29000 setps AND the loss on the train is:0.7369776964187622\n",
      "0.7224\n",
      "the 29500 setps AND the loss on the train is:0.927783727645874\n",
      "0.642\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "w1=tf.Variable(tf.random_normal([784,50]))\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=tf.Variable(tf.random_normal([50,50]))\n",
    "b2=tf.Variable(tf.random_normal([50]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.relu(logits2)\n",
    "\n",
    "w3=tf.Variable(tf.random_normal([50,50]))\n",
    "b3=tf.Variable(tf.random_normal([50]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "o3=tf.nn.relu(logits3)\n",
    "\n",
    "w4=tf.Variable(tf.random_normal([50,10]))\n",
    "b4=tf.Variable(tf.random_normal([10]))\n",
    "logits4=tf.matmul(o3,w4)+b4\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits4))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits4,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.添加正则化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.1 添加L2正则，上述单隐层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:253.0714874267578\n",
      "0.1574\n",
      "the 500 setps AND the loss on the train is:183.41183471679688\n",
      "0.6719\n",
      "the 1000 setps AND the loss on the train is:164.4830780029297\n",
      "0.7385\n",
      "the 1500 setps AND the loss on the train is:147.0012969970703\n",
      "0.7662\n",
      "the 2000 setps AND the loss on the train is:133.55621337890625\n",
      "0.7855\n",
      "the 2500 setps AND the loss on the train is:120.1465072631836\n",
      "0.7958\n",
      "the 3000 setps AND the loss on the train is:108.605712890625\n",
      "0.8061\n",
      "the 3500 setps AND the loss on the train is:98.25159454345703\n",
      "0.814\n",
      "the 4000 setps AND the loss on the train is:88.97042846679688\n",
      "0.8204\n",
      "the 4500 setps AND the loss on the train is:80.63278198242188\n",
      "0.8297\n",
      "the 5000 setps AND the loss on the train is:73.0064697265625\n",
      "0.8366\n",
      "the 5500 setps AND the loss on the train is:65.9490737915039\n",
      "0.8424\n",
      "the 6000 setps AND the loss on the train is:59.652740478515625\n",
      "0.8484\n",
      "the 6500 setps AND the loss on the train is:53.89625549316406\n",
      "0.8537\n",
      "the 7000 setps AND the loss on the train is:48.87160110473633\n",
      "0.8568\n",
      "the 7500 setps AND the loss on the train is:44.447120666503906\n",
      "0.8642\n",
      "the 8000 setps AND the loss on the train is:40.02593231201172\n",
      "0.8696\n",
      "the 8500 setps AND the loss on the train is:36.33076095581055\n",
      "0.8735\n",
      "the 9000 setps AND the loss on the train is:33.066070556640625\n",
      "0.8779\n",
      "the 9500 setps AND the loss on the train is:29.78158950805664\n",
      "0.8794\n",
      "the 10000 setps AND the loss on the train is:27.009050369262695\n",
      "0.8841\n",
      "the 10500 setps AND the loss on the train is:24.497713088989258\n",
      "0.8869\n",
      "the 11000 setps AND the loss on the train is:22.289478302001953\n",
      "0.8893\n",
      "the 11500 setps AND the loss on the train is:20.140409469604492\n",
      "0.8924\n",
      "the 12000 setps AND the loss on the train is:18.306407928466797\n",
      "0.8953\n",
      "the 12500 setps AND the loss on the train is:16.58625030517578\n",
      "0.8953\n",
      "the 13000 setps AND the loss on the train is:15.210793495178223\n",
      "0.8983\n",
      "the 13500 setps AND the loss on the train is:13.748160362243652\n",
      "0.9006\n",
      "the 14000 setps AND the loss on the train is:12.418107032775879\n",
      "0.9026\n",
      "the 14500 setps AND the loss on the train is:11.205681800842285\n",
      "0.9035\n",
      "the 15000 setps AND the loss on the train is:10.270745277404785\n",
      "0.9055\n",
      "the 15500 setps AND the loss on the train is:9.255990982055664\n",
      "0.9081\n",
      "the 16000 setps AND the loss on the train is:8.362109184265137\n",
      "0.9097\n",
      "the 16500 setps AND the loss on the train is:7.693389892578125\n",
      "0.9125\n",
      "the 17000 setps AND the loss on the train is:7.012822151184082\n",
      "0.9123\n",
      "the 17500 setps AND the loss on the train is:6.405743598937988\n",
      "0.9134\n",
      "the 18000 setps AND the loss on the train is:5.844543933868408\n",
      "0.9153\n",
      "the 18500 setps AND the loss on the train is:5.346476078033447\n",
      "0.917\n",
      "the 19000 setps AND the loss on the train is:4.831306457519531\n",
      "0.9171\n",
      "the 19500 setps AND the loss on the train is:4.461765289306641\n",
      "0.919\n",
      "the 20000 setps AND the loss on the train is:4.00512170791626\n",
      "0.9205\n",
      "the 20500 setps AND the loss on the train is:3.759939432144165\n",
      "0.9205\n",
      "the 21000 setps AND the loss on the train is:3.434849977493286\n",
      "0.9215\n",
      "the 21500 setps AND the loss on the train is:3.1459600925445557\n",
      "0.9218\n",
      "the 22000 setps AND the loss on the train is:2.9654717445373535\n",
      "0.9224\n",
      "the 22500 setps AND the loss on the train is:2.6278319358825684\n",
      "0.9237\n",
      "the 23000 setps AND the loss on the train is:2.4021053314208984\n",
      "0.9238\n",
      "the 23500 setps AND the loss on the train is:2.3552098274230957\n",
      "0.9257\n",
      "the 24000 setps AND the loss on the train is:2.242771863937378\n",
      "0.9252\n",
      "the 24500 setps AND the loss on the train is:1.8860714435577393\n",
      "0.9269\n",
      "the 25000 setps AND the loss on the train is:1.8308374881744385\n",
      "0.9264\n",
      "the 25500 setps AND the loss on the train is:1.7403465509414673\n",
      "0.9276\n",
      "the 26000 setps AND the loss on the train is:1.6065399646759033\n",
      "0.9282\n",
      "the 26500 setps AND the loss on the train is:1.5015559196472168\n",
      "0.9282\n",
      "the 27000 setps AND the loss on the train is:1.3273578882217407\n",
      "0.9285\n",
      "the 27500 setps AND the loss on the train is:1.3231806755065918\n",
      "0.9289\n",
      "the 28000 setps AND the loss on the train is:1.157576560974121\n",
      "0.9292\n",
      "the 28500 setps AND the loss on the train is:1.2157630920410156\n",
      "0.9294\n",
      "the 29000 setps AND the loss on the train is:1.0862410068511963\n",
      "0.93\n",
      "the 29500 setps AND the loss on the train is:1.0539112091064453\n",
      "0.9296\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,50],0.01)\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=get_weight([50,10],0.01)\n",
    "b2=tf.Variable(tf.random_normal([10]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits2))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits2,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.2 添加L1正则，上述单隐层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:568.1254272460938\n",
      "0.1104\n",
      "the 500 setps AND the loss on the train is:479.1046447753906\n",
      "0.6021\n",
      "the 1000 setps AND the loss on the train is:442.05218505859375\n",
      "0.6754\n",
      "the 1500 setps AND the loss on the train is:407.7109375\n",
      "0.7059\n",
      "the 2000 setps AND the loss on the train is:376.06915283203125\n",
      "0.7257\n",
      "the 2500 setps AND the loss on the train is:346.331787109375\n",
      "0.7391\n",
      "the 3000 setps AND the loss on the train is:319.5576477050781\n",
      "0.7479\n",
      "the 3500 setps AND the loss on the train is:294.2018737792969\n",
      "0.758\n",
      "the 4000 setps AND the loss on the train is:270.81085205078125\n",
      "0.7649\n",
      "the 4500 setps AND the loss on the train is:249.17909240722656\n",
      "0.7712\n",
      "the 5000 setps AND the loss on the train is:228.7797393798828\n",
      "0.7768\n",
      "the 5500 setps AND the loss on the train is:210.0352783203125\n",
      "0.7848\n",
      "the 6000 setps AND the loss on the train is:192.6005401611328\n",
      "0.7902\n",
      "the 6500 setps AND the loss on the train is:176.35833740234375\n",
      "0.7935\n",
      "the 7000 setps AND the loss on the train is:161.542236328125\n",
      "0.7996\n",
      "the 7500 setps AND the loss on the train is:147.4813232421875\n",
      "0.8042\n",
      "the 8000 setps AND the loss on the train is:134.64398193359375\n",
      "0.8032\n",
      "the 8500 setps AND the loss on the train is:122.73944854736328\n",
      "0.8065\n",
      "the 9000 setps AND the loss on the train is:111.86251831054688\n",
      "0.8106\n",
      "the 9500 setps AND the loss on the train is:101.89376068115234\n",
      "0.8133\n",
      "the 10000 setps AND the loss on the train is:92.46727752685547\n",
      "0.8139\n",
      "the 10500 setps AND the loss on the train is:84.07974243164062\n",
      "0.817\n",
      "the 11000 setps AND the loss on the train is:76.28914642333984\n",
      "0.8201\n",
      "the 11500 setps AND the loss on the train is:69.13866424560547\n",
      "0.8187\n",
      "the 12000 setps AND the loss on the train is:62.70856857299805\n",
      "0.8216\n",
      "the 12500 setps AND the loss on the train is:56.431373596191406\n",
      "0.8231\n",
      "the 13000 setps AND the loss on the train is:51.069820404052734\n",
      "0.8208\n",
      "the 13500 setps AND the loss on the train is:46.03409194946289\n",
      "0.8207\n",
      "the 14000 setps AND the loss on the train is:41.55882263183594\n",
      "0.8192\n",
      "the 14500 setps AND the loss on the train is:37.29524612426758\n",
      "0.8209\n",
      "the 15000 setps AND the loss on the train is:33.62028503417969\n",
      "0.8184\n",
      "the 15500 setps AND the loss on the train is:30.278579711914062\n",
      "0.8193\n",
      "the 16000 setps AND the loss on the train is:27.113954544067383\n",
      "0.8231\n",
      "the 16500 setps AND the loss on the train is:24.40880012512207\n",
      "0.8225\n",
      "the 17000 setps AND the loss on the train is:21.9454402923584\n",
      "0.8241\n",
      "the 17500 setps AND the loss on the train is:19.608121871948242\n",
      "0.8235\n",
      "the 18000 setps AND the loss on the train is:17.665210723876953\n",
      "0.8241\n",
      "the 18500 setps AND the loss on the train is:15.844304084777832\n",
      "0.8227\n",
      "the 19000 setps AND the loss on the train is:14.216414451599121\n",
      "0.8223\n",
      "the 19500 setps AND the loss on the train is:12.70675277709961\n",
      "0.8231\n",
      "the 20000 setps AND the loss on the train is:11.575310707092285\n",
      "0.8258\n",
      "the 20500 setps AND the loss on the train is:10.291272163391113\n",
      "0.8261\n",
      "the 21000 setps AND the loss on the train is:9.165169715881348\n",
      "0.824\n",
      "the 21500 setps AND the loss on the train is:8.381525993347168\n",
      "0.8267\n",
      "the 22000 setps AND the loss on the train is:7.549463272094727\n",
      "0.8259\n",
      "the 22500 setps AND the loss on the train is:6.94855260848999\n",
      "0.826\n",
      "the 23000 setps AND the loss on the train is:6.375500679016113\n",
      "0.8271\n",
      "the 23500 setps AND the loss on the train is:5.812478542327881\n",
      "0.8276\n",
      "the 24000 setps AND the loss on the train is:5.204780578613281\n",
      "0.8271\n",
      "the 24500 setps AND the loss on the train is:4.836616516113281\n",
      "0.8264\n",
      "the 25000 setps AND the loss on the train is:4.496144771575928\n",
      "0.8278\n",
      "the 25500 setps AND the loss on the train is:4.065829277038574\n",
      "0.8282\n",
      "the 26000 setps AND the loss on the train is:3.9198522567749023\n",
      "0.8272\n",
      "the 26500 setps AND the loss on the train is:3.558138370513916\n",
      "0.8291\n",
      "the 27000 setps AND the loss on the train is:3.2843313217163086\n",
      "0.8288\n",
      "the 27500 setps AND the loss on the train is:3.2186145782470703\n",
      "0.829\n",
      "the 28000 setps AND the loss on the train is:2.8287458419799805\n",
      "0.8293\n",
      "the 28500 setps AND the loss on the train is:2.636589527130127\n",
      "0.8306\n",
      "the 29000 setps AND the loss on the train is:2.6079630851745605\n",
      "0.8323\n",
      "the 29500 setps AND the loss on the train is:2.4400458335876465\n",
      "0.8324\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l1_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,50],0.01)\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=get_weight([50,10],0.01)\n",
    "b2=tf.Variable(tf.random_normal([10]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits2))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits2,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.3 一个隐层，使用L1+L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:1605.677734375\n",
      "0.0902\n",
      "the 500 setps AND the loss on the train is:1367.752685546875\n",
      "0.6565\n",
      "the 1000 setps AND the loss on the train is:1208.8341064453125\n",
      "0.7238\n",
      "the 1500 setps AND the loss on the train is:1068.2862548828125\n",
      "0.75\n",
      "the 2000 setps AND the loss on the train is:941.8690795898438\n",
      "0.7658\n",
      "the 2500 setps AND the loss on the train is:828.788330078125\n",
      "0.7795\n",
      "the 3000 setps AND the loss on the train is:727.8175048828125\n",
      "0.7917\n",
      "the 3500 setps AND the loss on the train is:637.4598999023438\n",
      "0.7999\n",
      "the 4000 setps AND the loss on the train is:557.3673706054688\n",
      "0.8114\n",
      "the 4500 setps AND the loss on the train is:485.9946594238281\n",
      "0.8182\n",
      "the 5000 setps AND the loss on the train is:423.03302001953125\n",
      "0.8243\n",
      "the 5500 setps AND the loss on the train is:367.4884338378906\n",
      "0.8284\n",
      "the 6000 setps AND the loss on the train is:318.81121826171875\n",
      "0.8304\n",
      "the 6500 setps AND the loss on the train is:276.1889953613281\n",
      "0.8338\n",
      "the 7000 setps AND the loss on the train is:239.47067260742188\n",
      "0.8364\n",
      "the 7500 setps AND the loss on the train is:207.67176818847656\n",
      "0.8362\n",
      "the 8000 setps AND the loss on the train is:180.06007385253906\n",
      "0.8361\n",
      "the 8500 setps AND the loss on the train is:156.72776794433594\n",
      "0.8359\n",
      "the 9000 setps AND the loss on the train is:136.7122802734375\n",
      "0.8335\n",
      "the 9500 setps AND the loss on the train is:119.56580352783203\n",
      "0.8304\n",
      "the 10000 setps AND the loss on the train is:105.11769104003906\n",
      "0.8257\n",
      "the 10500 setps AND the loss on the train is:92.62419891357422\n",
      "0.8229\n",
      "the 11000 setps AND the loss on the train is:82.18379974365234\n",
      "0.8231\n",
      "the 11500 setps AND the loss on the train is:73.07738494873047\n",
      "0.8211\n",
      "the 12000 setps AND the loss on the train is:65.16996765136719\n",
      "0.8203\n",
      "the 12500 setps AND the loss on the train is:58.52264404296875\n",
      "0.8223\n",
      "the 13000 setps AND the loss on the train is:52.44097900390625\n",
      "0.8269\n",
      "the 13500 setps AND the loss on the train is:47.14527893066406\n",
      "0.8284\n",
      "the 14000 setps AND the loss on the train is:42.44706726074219\n",
      "0.8294\n",
      "the 14500 setps AND the loss on the train is:38.11791229248047\n",
      "0.8305\n",
      "the 15000 setps AND the loss on the train is:34.367218017578125\n",
      "0.8312\n",
      "the 15500 setps AND the loss on the train is:30.845571517944336\n",
      "0.8324\n",
      "the 16000 setps AND the loss on the train is:27.805397033691406\n",
      "0.8345\n",
      "the 16500 setps AND the loss on the train is:24.90718650817871\n",
      "0.8381\n",
      "the 17000 setps AND the loss on the train is:22.43887710571289\n",
      "0.8395\n",
      "the 17500 setps AND the loss on the train is:20.23399543762207\n",
      "0.8384\n",
      "the 18000 setps AND the loss on the train is:18.049636840820312\n",
      "0.8412\n",
      "the 18500 setps AND the loss on the train is:16.257524490356445\n",
      "0.8389\n",
      "the 19000 setps AND the loss on the train is:14.505006790161133\n",
      "0.8361\n",
      "the 19500 setps AND the loss on the train is:13.090561866760254\n",
      "0.8366\n",
      "the 20000 setps AND the loss on the train is:11.814314842224121\n",
      "0.8391\n",
      "the 20500 setps AND the loss on the train is:10.653946876525879\n",
      "0.8377\n",
      "the 21000 setps AND the loss on the train is:9.556807518005371\n",
      "0.8382\n",
      "the 21500 setps AND the loss on the train is:8.530987739562988\n",
      "0.8383\n",
      "the 22000 setps AND the loss on the train is:7.676921844482422\n",
      "0.8378\n",
      "the 22500 setps AND the loss on the train is:6.9684224128723145\n",
      "0.838\n",
      "the 23000 setps AND the loss on the train is:6.472383499145508\n",
      "0.8361\n",
      "the 23500 setps AND the loss on the train is:5.868244171142578\n",
      "0.8373\n",
      "the 24000 setps AND the loss on the train is:5.159431457519531\n",
      "0.8348\n",
      "the 24500 setps AND the loss on the train is:4.855175971984863\n",
      "0.8362\n",
      "the 25000 setps AND the loss on the train is:4.465419292449951\n",
      "0.8346\n",
      "the 25500 setps AND the loss on the train is:4.262864589691162\n",
      "0.8351\n",
      "the 26000 setps AND the loss on the train is:3.9017369747161865\n",
      "0.8366\n",
      "the 26500 setps AND the loss on the train is:3.437068462371826\n",
      "0.836\n",
      "the 27000 setps AND the loss on the train is:3.279172658920288\n",
      "0.8372\n",
      "the 27500 setps AND the loss on the train is:3.0950300693511963\n",
      "0.8349\n",
      "the 28000 setps AND the loss on the train is:2.867459774017334\n",
      "0.8354\n",
      "the 28500 setps AND the loss on the train is:2.746809720993042\n",
      "0.8362\n",
      "the 29000 setps AND the loss on the train is:2.5918142795562744\n",
      "0.8349\n",
      "the 29500 setps AND the loss on the train is:2.4348998069763184\n",
      "0.8359\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l1_regularizer(lambd)(w))\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,50],0.01)\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=get_weight([50,10],0.01)\n",
    "b2=tf.Variable(tf.random_normal([10]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits2))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits2,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4添加两个隐层，使用L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:2427.401611328125\n",
      "0.0849\n",
      "the 500 setps AND the loss on the train is:2000.2740478515625\n",
      "0.66\n",
      "the 1000 setps AND the loss on the train is:1802.1534423828125\n",
      "0.6796\n",
      "the 1500 setps AND the loss on the train is:1623.3712158203125\n",
      "0.704\n",
      "the 2000 setps AND the loss on the train is:1460.84765625\n",
      "0.7177\n",
      "the 2500 setps AND the loss on the train is:1312.93701171875\n",
      "0.7483\n",
      "the 3000 setps AND the loss on the train is:1178.8463134765625\n",
      "0.7667\n",
      "the 3500 setps AND the loss on the train is:1056.796875\n",
      "0.7789\n",
      "the 4000 setps AND the loss on the train is:946.909912109375\n",
      "0.7839\n",
      "the 4500 setps AND the loss on the train is:846.8423461914062\n",
      "0.7802\n",
      "the 5000 setps AND the loss on the train is:756.8070068359375\n",
      "0.7866\n",
      "the 5500 setps AND the loss on the train is:676.1943969726562\n",
      "0.7952\n",
      "the 6000 setps AND the loss on the train is:603.3175659179688\n",
      "0.7942\n",
      "the 6500 setps AND the loss on the train is:538.200439453125\n",
      "0.7888\n",
      "the 7000 setps AND the loss on the train is:480.2385559082031\n",
      "0.802\n",
      "the 7500 setps AND the loss on the train is:428.35345458984375\n",
      "0.7985\n",
      "the 8000 setps AND the loss on the train is:382.4910583496094\n",
      "0.8039\n",
      "the 8500 setps AND the loss on the train is:341.5852355957031\n",
      "0.8084\n",
      "the 9000 setps AND the loss on the train is:305.1634521484375\n",
      "0.8053\n",
      "the 9500 setps AND the loss on the train is:273.0199890136719\n",
      "0.8087\n",
      "the 10000 setps AND the loss on the train is:244.40869140625\n",
      "0.8078\n",
      "the 10500 setps AND the loss on the train is:219.38296508789062\n",
      "0.8098\n",
      "the 11000 setps AND the loss on the train is:196.51223754882812\n",
      "0.8143\n",
      "the 11500 setps AND the loss on the train is:176.59481811523438\n",
      "0.8157\n",
      "the 12000 setps AND the loss on the train is:158.39999389648438\n",
      "0.8227\n",
      "the 12500 setps AND the loss on the train is:142.3256378173828\n",
      "0.8212\n",
      "the 13000 setps AND the loss on the train is:127.98148345947266\n",
      "0.8214\n",
      "the 13500 setps AND the loss on the train is:114.92980194091797\n",
      "0.8219\n",
      "the 14000 setps AND the loss on the train is:102.99298858642578\n",
      "0.819\n",
      "the 14500 setps AND the loss on the train is:92.21504211425781\n",
      "0.8187\n",
      "the 15000 setps AND the loss on the train is:82.7738037109375\n",
      "0.8215\n",
      "the 15500 setps AND the loss on the train is:73.86571502685547\n",
      "0.8199\n",
      "the 16000 setps AND the loss on the train is:66.15066528320312\n",
      "0.8209\n",
      "the 16500 setps AND the loss on the train is:58.991600036621094\n",
      "0.8206\n",
      "the 17000 setps AND the loss on the train is:52.658973693847656\n",
      "0.8206\n",
      "the 17500 setps AND the loss on the train is:46.9918098449707\n",
      "0.8223\n",
      "the 18000 setps AND the loss on the train is:41.71844482421875\n",
      "0.824\n",
      "the 18500 setps AND the loss on the train is:37.104061126708984\n",
      "0.8223\n",
      "the 19000 setps AND the loss on the train is:32.88292694091797\n",
      "0.8208\n",
      "the 19500 setps AND the loss on the train is:29.196517944335938\n",
      "0.8219\n",
      "the 20000 setps AND the loss on the train is:25.875581741333008\n",
      "0.8222\n",
      "the 20500 setps AND the loss on the train is:23.040224075317383\n",
      "0.8247\n",
      "the 21000 setps AND the loss on the train is:20.411218643188477\n",
      "0.8254\n",
      "the 21500 setps AND the loss on the train is:17.94449806213379\n",
      "0.821\n",
      "the 22000 setps AND the loss on the train is:15.84579849243164\n",
      "0.8229\n",
      "the 22500 setps AND the loss on the train is:14.10213565826416\n",
      "0.8237\n",
      "the 23000 setps AND the loss on the train is:12.39694595336914\n",
      "0.8269\n",
      "the 23500 setps AND the loss on the train is:11.14794921875\n",
      "0.8267\n",
      "the 24000 setps AND the loss on the train is:10.039953231811523\n",
      "0.8249\n",
      "the 24500 setps AND the loss on the train is:8.868536949157715\n",
      "0.8289\n",
      "the 25000 setps AND the loss on the train is:8.067397117614746\n",
      "0.8262\n",
      "the 25500 setps AND the loss on the train is:7.11489200592041\n",
      "0.8293\n",
      "the 26000 setps AND the loss on the train is:6.386382579803467\n",
      "0.8304\n",
      "the 26500 setps AND the loss on the train is:5.822288990020752\n",
      "0.8305\n",
      "the 27000 setps AND the loss on the train is:5.245805740356445\n",
      "0.8347\n",
      "the 27500 setps AND the loss on the train is:4.746090888977051\n",
      "0.8303\n",
      "the 28000 setps AND the loss on the train is:4.232181072235107\n",
      "0.8387\n",
      "the 28500 setps AND the loss on the train is:3.7999649047851562\n",
      "0.8402\n",
      "the 29000 setps AND the loss on the train is:3.511880397796631\n",
      "0.8413\n",
      "the 29500 setps AND the loss on the train is:3.2215447425842285\n",
      "0.8429\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l1_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,50],0.01)\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=get_weight([50,50],0.01)\n",
    "b2=tf.Variable(tf.random_normal([50]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.relu(logits2)\n",
    "\n",
    "w3=get_weight([50,10],0.01)\n",
    "b3=tf.Variable(tf.random_normal([10]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits3))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits3,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4添加两个隐层，使用L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:2666.333984375\n",
      "0.0925\n",
      "the 500 setps AND the loss on the train is:2187.075927734375\n",
      "0.6269\n",
      "the 1000 setps AND the loss on the train is:1972.6500244140625\n",
      "0.6779\n",
      "the 1500 setps AND the loss on the train is:1778.0498046875\n",
      "0.7014\n",
      "the 2000 setps AND the loss on the train is:1600.876220703125\n",
      "0.7299\n",
      "the 2500 setps AND the loss on the train is:1439.7933349609375\n",
      "0.7451\n",
      "the 3000 setps AND the loss on the train is:1293.254150390625\n",
      "0.7681\n",
      "the 3500 setps AND the loss on the train is:1160.7216796875\n",
      "0.7889\n",
      "the 4000 setps AND the loss on the train is:1040.4454345703125\n",
      "0.8009\n",
      "the 4500 setps AND the loss on the train is:931.969482421875\n",
      "0.8101\n",
      "the 5000 setps AND the loss on the train is:834.2615966796875\n",
      "0.8231\n",
      "the 5500 setps AND the loss on the train is:746.1429443359375\n",
      "0.8324\n",
      "the 6000 setps AND the loss on the train is:667.2902221679688\n",
      "0.8391\n",
      "the 6500 setps AND the loss on the train is:595.9862670898438\n",
      "0.8461\n",
      "the 7000 setps AND the loss on the train is:532.61181640625\n",
      "0.8531\n",
      "the 7500 setps AND the loss on the train is:475.9988098144531\n",
      "0.8578\n",
      "the 8000 setps AND the loss on the train is:425.57476806640625\n",
      "0.8617\n",
      "the 8500 setps AND the loss on the train is:380.9096984863281\n",
      "0.8679\n",
      "the 9000 setps AND the loss on the train is:340.9586486816406\n",
      "0.8716\n",
      "the 9500 setps AND the loss on the train is:305.5866394042969\n",
      "0.8774\n",
      "the 10000 setps AND the loss on the train is:274.0561218261719\n",
      "0.878\n",
      "the 10500 setps AND the loss on the train is:246.0211944580078\n",
      "0.8828\n",
      "the 11000 setps AND the loss on the train is:221.00274658203125\n",
      "0.8864\n",
      "the 11500 setps AND the loss on the train is:198.61639404296875\n",
      "0.8905\n",
      "the 12000 setps AND the loss on the train is:178.56544494628906\n",
      "0.8928\n",
      "the 12500 setps AND the loss on the train is:160.38792419433594\n",
      "0.8982\n",
      "the 13000 setps AND the loss on the train is:144.15667724609375\n",
      "0.9005\n",
      "the 13500 setps AND the loss on the train is:129.49278259277344\n",
      "0.9027\n",
      "the 14000 setps AND the loss on the train is:116.13037109375\n",
      "0.904\n",
      "the 14500 setps AND the loss on the train is:104.09613037109375\n",
      "0.9051\n",
      "the 15000 setps AND the loss on the train is:93.30260467529297\n",
      "0.9083\n",
      "the 15500 setps AND the loss on the train is:83.48905181884766\n",
      "0.9086\n",
      "the 16000 setps AND the loss on the train is:74.5770492553711\n",
      "0.9074\n",
      "the 16500 setps AND the loss on the train is:66.57091522216797\n",
      "0.9146\n",
      "the 17000 setps AND the loss on the train is:59.250083923339844\n",
      "0.9143\n",
      "the 17500 setps AND the loss on the train is:52.665008544921875\n",
      "0.9171\n",
      "the 18000 setps AND the loss on the train is:46.846527099609375\n",
      "0.9156\n",
      "the 18500 setps AND the loss on the train is:41.57820510864258\n",
      "0.9193\n",
      "the 19000 setps AND the loss on the train is:36.81508255004883\n",
      "0.921\n",
      "the 19500 setps AND the loss on the train is:32.6489372253418\n",
      "0.9218\n",
      "the 20000 setps AND the loss on the train is:28.81987953186035\n",
      "0.9239\n",
      "the 20500 setps AND the loss on the train is:25.5185546875\n",
      "0.9241\n",
      "the 21000 setps AND the loss on the train is:22.428178787231445\n",
      "0.9245\n",
      "the 21500 setps AND the loss on the train is:19.86611557006836\n",
      "0.9249\n",
      "the 22000 setps AND the loss on the train is:17.578561782836914\n",
      "0.9246\n",
      "the 22500 setps AND the loss on the train is:15.409961700439453\n",
      "0.9265\n",
      "the 23000 setps AND the loss on the train is:13.586373329162598\n",
      "0.9261\n",
      "the 23500 setps AND the loss on the train is:12.02821159362793\n",
      "0.9266\n",
      "the 24000 setps AND the loss on the train is:10.556731224060059\n",
      "0.9276\n",
      "the 24500 setps AND the loss on the train is:9.412215232849121\n",
      "0.9287\n",
      "the 25000 setps AND the loss on the train is:8.333637237548828\n",
      "0.9285\n",
      "the 25500 setps AND the loss on the train is:7.234864234924316\n",
      "0.9285\n",
      "the 26000 setps AND the loss on the train is:6.421027183532715\n",
      "0.9295\n",
      "the 26500 setps AND the loss on the train is:5.629868507385254\n",
      "0.9306\n",
      "the 27000 setps AND the loss on the train is:5.048654079437256\n",
      "0.9309\n",
      "the 27500 setps AND the loss on the train is:4.449702262878418\n",
      "0.9307\n",
      "the 28000 setps AND the loss on the train is:3.993197441101074\n",
      "0.9323\n",
      "the 28500 setps AND the loss on the train is:3.4954936504364014\n",
      "0.9315\n",
      "the 29000 setps AND the loss on the train is:3.1609206199645996\n",
      "0.9315\n",
      "the 29500 setps AND the loss on the train is:2.925520896911621\n",
      "0.9328\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,50],0.01)\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=get_weight([50,50],0.01)\n",
    "b2=tf.Variable(tf.random_normal([50]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.relu(logits2)\n",
    "\n",
    "w3=get_weight([50,10],0.01)\n",
    "b3=tf.Variable(tf.random_normal([10]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits3))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits3,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(30000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3.4.1 同3.4，只不过把迭代次数增加10倍"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:2894.699951171875\n",
      "0.0693\n",
      "the 500 setps AND the loss on the train is:2387.830322265625\n",
      "0.6756\n",
      "the 1000 setps AND the loss on the train is:2153.77734375\n",
      "0.6667\n",
      "the 1500 setps AND the loss on the train is:1942.058349609375\n",
      "0.6752\n",
      "the 2000 setps AND the loss on the train is:1749.5531005859375\n",
      "0.726\n",
      "the 2500 setps AND the loss on the train is:1574.4254150390625\n",
      "0.7414\n",
      "the 3000 setps AND the loss on the train is:1415.80615234375\n",
      "0.7607\n",
      "the 3500 setps AND the loss on the train is:1271.4560546875\n",
      "0.7818\n",
      "the 4000 setps AND the loss on the train is:1141.08154296875\n",
      "0.7952\n",
      "the 4500 setps AND the loss on the train is:1022.9968872070312\n",
      "0.8136\n",
      "the 5000 setps AND the loss on the train is:916.747802734375\n",
      "0.8233\n",
      "the 5500 setps AND the loss on the train is:820.6237182617188\n",
      "0.8308\n",
      "the 6000 setps AND the loss on the train is:734.6162719726562\n",
      "0.837\n",
      "the 6500 setps AND the loss on the train is:657.3798828125\n",
      "0.8478\n",
      "the 7000 setps AND the loss on the train is:588.158203125\n",
      "0.8586\n",
      "the 7500 setps AND the loss on the train is:526.0409545898438\n",
      "0.8628\n",
      "the 8000 setps AND the loss on the train is:470.7952575683594\n",
      "0.8694\n",
      "the 8500 setps AND the loss on the train is:421.5655822753906\n",
      "0.8765\n",
      "the 9000 setps AND the loss on the train is:377.81103515625\n",
      "0.8801\n",
      "the 9500 setps AND the loss on the train is:338.7708435058594\n",
      "0.8822\n",
      "the 10000 setps AND the loss on the train is:304.2951965332031\n",
      "0.8885\n",
      "the 10500 setps AND the loss on the train is:273.17926025390625\n",
      "0.8933\n",
      "the 11000 setps AND the loss on the train is:245.67770385742188\n",
      "0.896\n",
      "the 11500 setps AND the loss on the train is:220.82562255859375\n",
      "0.9009\n",
      "the 12000 setps AND the loss on the train is:198.7627410888672\n",
      "0.9033\n",
      "the 12500 setps AND the loss on the train is:178.95895385742188\n",
      "0.9055\n",
      "the 13000 setps AND the loss on the train is:160.91424560546875\n",
      "0.9074\n",
      "the 13500 setps AND the loss on the train is:144.74258422851562\n",
      "0.9104\n",
      "the 14000 setps AND the loss on the train is:130.09046936035156\n",
      "0.9127\n",
      "the 14500 setps AND the loss on the train is:116.871826171875\n",
      "0.9155\n",
      "the 15000 setps AND the loss on the train is:104.63028717041016\n",
      "0.9154\n",
      "the 15500 setps AND the loss on the train is:93.79523468017578\n",
      "0.918\n",
      "the 16000 setps AND the loss on the train is:84.03565216064453\n",
      "0.9204\n",
      "the 16500 setps AND the loss on the train is:74.90850830078125\n",
      "0.9216\n",
      "the 17000 setps AND the loss on the train is:66.94474792480469\n",
      "0.9211\n",
      "the 17500 setps AND the loss on the train is:59.56877899169922\n",
      "0.9233\n",
      "the 18000 setps AND the loss on the train is:53.14836883544922\n",
      "0.9246\n",
      "the 18500 setps AND the loss on the train is:47.400352478027344\n",
      "0.9252\n",
      "the 19000 setps AND the loss on the train is:42.1338005065918\n",
      "0.9253\n",
      "the 19500 setps AND the loss on the train is:37.41823959350586\n",
      "0.9284\n",
      "the 20000 setps AND the loss on the train is:33.153533935546875\n",
      "0.9294\n",
      "the 20500 setps AND the loss on the train is:29.553455352783203\n",
      "0.9296\n",
      "the 21000 setps AND the loss on the train is:26.02659034729004\n",
      "0.9299\n",
      "the 21500 setps AND the loss on the train is:23.059877395629883\n",
      "0.9299\n",
      "the 22000 setps AND the loss on the train is:20.35265350341797\n",
      "0.9299\n",
      "the 22500 setps AND the loss on the train is:17.978654861450195\n",
      "0.9312\n",
      "the 23000 setps AND the loss on the train is:15.949555397033691\n",
      "0.9312\n",
      "the 23500 setps AND the loss on the train is:14.045281410217285\n",
      "0.9322\n",
      "the 24000 setps AND the loss on the train is:12.437363624572754\n",
      "0.9326\n",
      "the 24500 setps AND the loss on the train is:11.128820419311523\n",
      "0.9316\n",
      "the 25000 setps AND the loss on the train is:9.719070434570312\n",
      "0.9326\n",
      "the 25500 setps AND the loss on the train is:8.612348556518555\n",
      "0.9346\n",
      "the 26000 setps AND the loss on the train is:7.726587295532227\n",
      "0.9344\n",
      "the 26500 setps AND the loss on the train is:6.9217424392700195\n",
      "0.9345\n",
      "the 27000 setps AND the loss on the train is:6.177707195281982\n",
      "0.9357\n",
      "the 27500 setps AND the loss on the train is:5.501021862030029\n",
      "0.9363\n",
      "the 28000 setps AND the loss on the train is:4.845471382141113\n",
      "0.9358\n",
      "the 28500 setps AND the loss on the train is:4.394679069519043\n",
      "0.9359\n",
      "the 29000 setps AND the loss on the train is:3.9877095222473145\n",
      "0.936\n",
      "the 29500 setps AND the loss on the train is:3.482414484024048\n",
      "0.9365\n",
      "the 30000 setps AND the loss on the train is:3.1518027782440186\n",
      "0.9354\n",
      "the 30500 setps AND the loss on the train is:2.801156520843506\n",
      "0.9367\n",
      "the 31000 setps AND the loss on the train is:2.553894519805908\n",
      "0.9375\n",
      "the 31500 setps AND the loss on the train is:2.2958765029907227\n",
      "0.9378\n",
      "the 32000 setps AND the loss on the train is:2.111111879348755\n",
      "0.9382\n",
      "the 32500 setps AND the loss on the train is:2.0014641284942627\n",
      "0.9369\n",
      "the 33000 setps AND the loss on the train is:1.7476469278335571\n",
      "0.9372\n",
      "the 33500 setps AND the loss on the train is:1.669413447380066\n",
      "0.9394\n",
      "the 34000 setps AND the loss on the train is:1.5356004238128662\n",
      "0.9379\n",
      "the 34500 setps AND the loss on the train is:1.4445899724960327\n",
      "0.9382\n",
      "the 35000 setps AND the loss on the train is:1.275177240371704\n",
      "0.9391\n",
      "the 35500 setps AND the loss on the train is:1.2605940103530884\n",
      "0.9388\n",
      "the 36000 setps AND the loss on the train is:1.196518898010254\n",
      "0.9403\n",
      "the 36500 setps AND the loss on the train is:1.1454946994781494\n",
      "0.9411\n",
      "the 37000 setps AND the loss on the train is:1.0617915391921997\n",
      "0.94\n",
      "the 37500 setps AND the loss on the train is:1.0778491497039795\n",
      "0.9404\n",
      "the 38000 setps AND the loss on the train is:1.009101390838623\n",
      "0.9406\n",
      "the 38500 setps AND the loss on the train is:0.8974431753158569\n",
      "0.9408\n",
      "the 39000 setps AND the loss on the train is:0.9590725898742676\n",
      "0.9417\n",
      "the 39500 setps AND the loss on the train is:0.8655909895896912\n",
      "0.9415\n",
      "the 40000 setps AND the loss on the train is:0.7985863089561462\n",
      "0.9416\n",
      "the 40500 setps AND the loss on the train is:0.8267728090286255\n",
      "0.9424\n",
      "the 41000 setps AND the loss on the train is:0.7625294327735901\n",
      "0.942\n",
      "the 41500 setps AND the loss on the train is:0.8123429417610168\n",
      "0.9417\n",
      "the 42000 setps AND the loss on the train is:0.7908002138137817\n",
      "0.9417\n",
      "the 42500 setps AND the loss on the train is:0.7935842275619507\n",
      "0.9413\n",
      "the 43000 setps AND the loss on the train is:0.7514688372612\n",
      "0.9432\n",
      "the 43500 setps AND the loss on the train is:0.7589256763458252\n",
      "0.9417\n",
      "the 44000 setps AND the loss on the train is:0.6837300062179565\n",
      "0.9437\n",
      "the 44500 setps AND the loss on the train is:0.6634946465492249\n",
      "0.9432\n",
      "the 45000 setps AND the loss on the train is:0.6023246645927429\n",
      "0.9427\n",
      "the 45500 setps AND the loss on the train is:0.6701796650886536\n",
      "0.9423\n",
      "the 46000 setps AND the loss on the train is:0.6914075613021851\n",
      "0.9432\n",
      "the 46500 setps AND the loss on the train is:0.6368952989578247\n",
      "0.9429\n",
      "the 47000 setps AND the loss on the train is:0.714426577091217\n",
      "0.9428\n",
      "the 47500 setps AND the loss on the train is:0.6671768426895142\n",
      "0.9424\n",
      "the 48000 setps AND the loss on the train is:0.6276547908782959\n",
      "0.9441\n",
      "the 48500 setps AND the loss on the train is:0.7300337553024292\n",
      "0.9429\n",
      "the 49000 setps AND the loss on the train is:0.617691159248352\n",
      "0.9437\n",
      "the 49500 setps AND the loss on the train is:0.6869730949401855\n",
      "0.944\n",
      "the 50000 setps AND the loss on the train is:0.6460860967636108\n",
      "0.9447\n",
      "the 50500 setps AND the loss on the train is:0.7112249135971069\n",
      "0.9437\n",
      "the 51000 setps AND the loss on the train is:0.7114464044570923\n",
      "0.9447\n",
      "the 51500 setps AND the loss on the train is:0.6733824014663696\n",
      "0.9439\n",
      "the 52000 setps AND the loss on the train is:0.6197836399078369\n",
      "0.944\n",
      "the 52500 setps AND the loss on the train is:0.5844559669494629\n",
      "0.9443\n",
      "the 53000 setps AND the loss on the train is:0.7320863008499146\n",
      "0.9437\n",
      "the 53500 setps AND the loss on the train is:0.8153722286224365\n",
      "0.9455\n",
      "the 54000 setps AND the loss on the train is:0.6230922937393188\n",
      "0.9435\n",
      "the 54500 setps AND the loss on the train is:0.5443504452705383\n",
      "0.9445\n",
      "the 55000 setps AND the loss on the train is:0.5032548904418945\n",
      "0.9433\n",
      "the 55500 setps AND the loss on the train is:0.5974772572517395\n",
      "0.9442\n",
      "the 56000 setps AND the loss on the train is:0.6681016683578491\n",
      "0.9446\n",
      "the 56500 setps AND the loss on the train is:0.7798641920089722\n",
      "0.9451\n",
      "the 57000 setps AND the loss on the train is:0.5642810463905334\n",
      "0.9449\n",
      "the 57500 setps AND the loss on the train is:0.6591354608535767\n",
      "0.9453\n",
      "the 58000 setps AND the loss on the train is:0.645503044128418\n",
      "0.9457\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 58500 setps AND the loss on the train is:0.5584689974784851\n",
      "0.947\n",
      "the 59000 setps AND the loss on the train is:0.672429621219635\n",
      "0.9459\n",
      "the 59500 setps AND the loss on the train is:0.5947092175483704\n",
      "0.9451\n",
      "the 60000 setps AND the loss on the train is:0.7176923155784607\n",
      "0.9463\n",
      "the 60500 setps AND the loss on the train is:0.6272236704826355\n",
      "0.9448\n",
      "the 61000 setps AND the loss on the train is:0.6367537975311279\n",
      "0.9466\n",
      "the 61500 setps AND the loss on the train is:0.5923712849617004\n",
      "0.9457\n",
      "the 62000 setps AND the loss on the train is:0.5926653146743774\n",
      "0.946\n",
      "the 62500 setps AND the loss on the train is:0.5726847052574158\n",
      "0.9468\n",
      "the 63000 setps AND the loss on the train is:0.6134898662567139\n",
      "0.9458\n",
      "the 63500 setps AND the loss on the train is:0.6272127032279968\n",
      "0.9471\n",
      "the 64000 setps AND the loss on the train is:0.6505497694015503\n",
      "0.9458\n",
      "the 64500 setps AND the loss on the train is:0.5166029930114746\n",
      "0.9464\n",
      "the 65000 setps AND the loss on the train is:0.6039470434188843\n",
      "0.9467\n",
      "the 65500 setps AND the loss on the train is:0.6573280692100525\n",
      "0.9471\n",
      "the 66000 setps AND the loss on the train is:0.6280295848846436\n",
      "0.9456\n",
      "the 66500 setps AND the loss on the train is:0.597053587436676\n",
      "0.9462\n",
      "the 67000 setps AND the loss on the train is:0.606730043888092\n",
      "0.9464\n",
      "the 67500 setps AND the loss on the train is:0.5269681215286255\n",
      "0.9472\n",
      "the 68000 setps AND the loss on the train is:0.606033205986023\n",
      "0.9471\n",
      "the 68500 setps AND the loss on the train is:0.6662647724151611\n",
      "0.9475\n",
      "the 69000 setps AND the loss on the train is:0.6169006824493408\n",
      "0.9475\n",
      "the 69500 setps AND the loss on the train is:0.608525276184082\n",
      "0.9475\n",
      "the 70000 setps AND the loss on the train is:0.6948050260543823\n",
      "0.9475\n",
      "the 70500 setps AND the loss on the train is:0.5753548741340637\n",
      "0.9483\n",
      "the 71000 setps AND the loss on the train is:0.6119893789291382\n",
      "0.9475\n",
      "the 71500 setps AND the loss on the train is:0.6071202754974365\n",
      "0.9478\n",
      "the 72000 setps AND the loss on the train is:0.5311036109924316\n",
      "0.9481\n",
      "the 72500 setps AND the loss on the train is:0.5607912540435791\n",
      "0.9487\n",
      "the 73000 setps AND the loss on the train is:0.581483006477356\n",
      "0.9476\n",
      "the 73500 setps AND the loss on the train is:0.5629324913024902\n",
      "0.9477\n",
      "the 74000 setps AND the loss on the train is:0.6241234540939331\n",
      "0.9478\n",
      "the 74500 setps AND the loss on the train is:0.6132009029388428\n",
      "0.9484\n",
      "the 75000 setps AND the loss on the train is:0.647793710231781\n",
      "0.9476\n",
      "the 75500 setps AND the loss on the train is:0.6724293828010559\n",
      "0.9477\n",
      "the 76000 setps AND the loss on the train is:0.6279882788658142\n",
      "0.9485\n",
      "the 76500 setps AND the loss on the train is:0.6845365166664124\n",
      "0.9493\n",
      "the 77000 setps AND the loss on the train is:0.6069575548171997\n",
      "0.9477\n",
      "the 77500 setps AND the loss on the train is:0.5663887858390808\n",
      "0.9478\n",
      "the 78000 setps AND the loss on the train is:0.6083021759986877\n",
      "0.9486\n",
      "the 78500 setps AND the loss on the train is:0.6476434469223022\n",
      "0.9489\n",
      "the 79000 setps AND the loss on the train is:0.7002221941947937\n",
      "0.9485\n",
      "the 79500 setps AND the loss on the train is:0.6087352633476257\n",
      "0.9491\n",
      "the 80000 setps AND the loss on the train is:0.595974326133728\n",
      "0.949\n",
      "the 80500 setps AND the loss on the train is:0.5712748765945435\n",
      "0.9486\n",
      "the 81000 setps AND the loss on the train is:0.6928972005844116\n",
      "0.9487\n",
      "the 81500 setps AND the loss on the train is:0.5764527320861816\n",
      "0.9491\n",
      "the 82000 setps AND the loss on the train is:0.5934438705444336\n",
      "0.9491\n",
      "the 82500 setps AND the loss on the train is:0.5719088912010193\n",
      "0.9482\n",
      "the 83000 setps AND the loss on the train is:0.6666454076766968\n",
      "0.9488\n",
      "the 83500 setps AND the loss on the train is:0.5768736600875854\n",
      "0.9483\n",
      "the 84000 setps AND the loss on the train is:0.5373178720474243\n",
      "0.9498\n",
      "the 84500 setps AND the loss on the train is:0.561337947845459\n",
      "0.9478\n",
      "the 85000 setps AND the loss on the train is:0.5516411662101746\n",
      "0.9492\n",
      "the 85500 setps AND the loss on the train is:0.556179404258728\n",
      "0.9493\n",
      "the 86000 setps AND the loss on the train is:0.6504054069519043\n",
      "0.9487\n",
      "the 86500 setps AND the loss on the train is:0.6291397213935852\n",
      "0.9488\n",
      "the 87000 setps AND the loss on the train is:0.6037169694900513\n",
      "0.9492\n",
      "the 87500 setps AND the loss on the train is:0.5953191518783569\n",
      "0.9484\n",
      "the 88000 setps AND the loss on the train is:0.6588242053985596\n",
      "0.9483\n",
      "the 88500 setps AND the loss on the train is:0.6789318323135376\n",
      "0.9487\n",
      "the 89000 setps AND the loss on the train is:0.6324828863143921\n",
      "0.949\n",
      "the 89500 setps AND the loss on the train is:0.5722057223320007\n",
      "0.9486\n",
      "the 90000 setps AND the loss on the train is:0.608884334564209\n",
      "0.9497\n",
      "the 90500 setps AND the loss on the train is:0.6342806816101074\n",
      "0.9492\n",
      "the 91000 setps AND the loss on the train is:0.741362452507019\n",
      "0.948\n",
      "the 91500 setps AND the loss on the train is:0.5459854006767273\n",
      "0.9497\n",
      "the 92000 setps AND the loss on the train is:0.501269519329071\n",
      "0.9491\n",
      "the 92500 setps AND the loss on the train is:0.5894467830657959\n",
      "0.9497\n",
      "the 93000 setps AND the loss on the train is:0.5939921140670776\n",
      "0.9492\n",
      "the 93500 setps AND the loss on the train is:0.5532731413841248\n",
      "0.9494\n",
      "the 94000 setps AND the loss on the train is:0.5258805751800537\n",
      "0.9489\n",
      "the 94500 setps AND the loss on the train is:0.643097460269928\n",
      "0.9495\n",
      "the 95000 setps AND the loss on the train is:0.6107808351516724\n",
      "0.9494\n",
      "the 95500 setps AND the loss on the train is:0.588140606880188\n",
      "0.9486\n",
      "the 96000 setps AND the loss on the train is:0.6083338260650635\n",
      "0.9498\n",
      "the 96500 setps AND the loss on the train is:0.7058918476104736\n",
      "0.9488\n",
      "the 97000 setps AND the loss on the train is:0.5707963705062866\n",
      "0.9501\n",
      "the 97500 setps AND the loss on the train is:0.6618489027023315\n",
      "0.9485\n",
      "the 98000 setps AND the loss on the train is:0.5905463695526123\n",
      "0.9493\n",
      "the 98500 setps AND the loss on the train is:0.600757896900177\n",
      "0.9487\n",
      "the 99000 setps AND the loss on the train is:0.6102877259254456\n",
      "0.9483\n",
      "the 99500 setps AND the loss on the train is:0.5756576061248779\n",
      "0.949\n",
      "the 100000 setps AND the loss on the train is:0.5640847086906433\n",
      "0.9492\n",
      "the 100500 setps AND the loss on the train is:0.5574823617935181\n",
      "0.9495\n",
      "the 101000 setps AND the loss on the train is:0.5760582089424133\n",
      "0.9479\n",
      "the 101500 setps AND the loss on the train is:0.5357174277305603\n",
      "0.9497\n",
      "the 102000 setps AND the loss on the train is:0.5419013500213623\n",
      "0.95\n",
      "the 102500 setps AND the loss on the train is:0.5695886611938477\n",
      "0.9498\n",
      "the 103000 setps AND the loss on the train is:0.5978373289108276\n",
      "0.9497\n",
      "the 103500 setps AND the loss on the train is:0.5554527044296265\n",
      "0.949\n",
      "the 104000 setps AND the loss on the train is:0.5338457226753235\n",
      "0.9502\n",
      "the 104500 setps AND the loss on the train is:0.5894134640693665\n",
      "0.9497\n",
      "the 105000 setps AND the loss on the train is:0.5403339862823486\n",
      "0.9489\n",
      "the 105500 setps AND the loss on the train is:0.5366261005401611\n",
      "0.9503\n",
      "the 106000 setps AND the loss on the train is:0.6092709302902222\n",
      "0.9501\n",
      "the 106500 setps AND the loss on the train is:0.6084484457969666\n",
      "0.949\n",
      "the 107000 setps AND the loss on the train is:0.6754400730133057\n",
      "0.9489\n",
      "the 107500 setps AND the loss on the train is:0.5822544097900391\n",
      "0.9498\n",
      "the 108000 setps AND the loss on the train is:0.6607163548469543\n",
      "0.9496\n",
      "the 108500 setps AND the loss on the train is:0.5220664739608765\n",
      "0.9492\n",
      "the 109000 setps AND the loss on the train is:0.6093165874481201\n",
      "0.9496\n",
      "the 109500 setps AND the loss on the train is:0.6114664077758789\n",
      "0.9493\n",
      "the 110000 setps AND the loss on the train is:0.5254377722740173\n",
      "0.95\n",
      "the 110500 setps AND the loss on the train is:0.5423243641853333\n",
      "0.9511\n",
      "the 111000 setps AND the loss on the train is:0.5758654475212097\n",
      "0.9494\n",
      "the 111500 setps AND the loss on the train is:0.5350257158279419\n",
      "0.9489\n",
      "the 112000 setps AND the loss on the train is:0.6512448191642761\n",
      "0.9497\n",
      "the 112500 setps AND the loss on the train is:0.527259349822998\n",
      "0.9502\n",
      "the 113000 setps AND the loss on the train is:0.5609397888183594\n",
      "0.9498\n",
      "the 113500 setps AND the loss on the train is:0.563281774520874\n",
      "0.9499\n",
      "the 114000 setps AND the loss on the train is:0.5851348042488098\n",
      "0.9489\n",
      "the 114500 setps AND the loss on the train is:0.5583429336547852\n",
      "0.9488\n",
      "the 115000 setps AND the loss on the train is:0.522875964641571\n",
      "0.9504\n",
      "the 115500 setps AND the loss on the train is:0.664020299911499\n",
      "0.9495\n",
      "the 116000 setps AND the loss on the train is:0.5672375559806824\n",
      "0.9501\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 116500 setps AND the loss on the train is:0.6339830756187439\n",
      "0.9487\n",
      "the 117000 setps AND the loss on the train is:0.6307917833328247\n",
      "0.9496\n",
      "the 117500 setps AND the loss on the train is:0.5679863691329956\n",
      "0.9499\n",
      "the 118000 setps AND the loss on the train is:0.5126785635948181\n",
      "0.9501\n",
      "the 118500 setps AND the loss on the train is:0.5466687679290771\n",
      "0.95\n",
      "the 119000 setps AND the loss on the train is:0.6279670000076294\n",
      "0.9499\n",
      "the 119500 setps AND the loss on the train is:0.5987111926078796\n",
      "0.95\n",
      "the 120000 setps AND the loss on the train is:0.5814155340194702\n",
      "0.9494\n",
      "the 120500 setps AND the loss on the train is:0.6037892699241638\n",
      "0.95\n",
      "the 121000 setps AND the loss on the train is:0.5654520392417908\n",
      "0.9504\n",
      "the 121500 setps AND the loss on the train is:0.5161345601081848\n",
      "0.95\n",
      "the 122000 setps AND the loss on the train is:0.6286090016365051\n",
      "0.9497\n",
      "the 122500 setps AND the loss on the train is:0.6246615648269653\n",
      "0.9506\n",
      "the 123000 setps AND the loss on the train is:0.6756899356842041\n",
      "0.9503\n",
      "the 123500 setps AND the loss on the train is:0.6276418566703796\n",
      "0.9504\n",
      "the 124000 setps AND the loss on the train is:0.5554254055023193\n",
      "0.9504\n",
      "the 124500 setps AND the loss on the train is:0.5495278239250183\n",
      "0.95\n",
      "the 125000 setps AND the loss on the train is:0.5281093716621399\n",
      "0.9497\n",
      "the 125500 setps AND the loss on the train is:0.6324435472488403\n",
      "0.9501\n",
      "the 126000 setps AND the loss on the train is:0.7188405990600586\n",
      "0.95\n",
      "the 126500 setps AND the loss on the train is:0.6703051328659058\n",
      "0.9496\n",
      "the 127000 setps AND the loss on the train is:0.5556662082672119\n",
      "0.9505\n",
      "the 127500 setps AND the loss on the train is:0.5848454833030701\n",
      "0.9494\n",
      "the 128000 setps AND the loss on the train is:0.5474052429199219\n",
      "0.9502\n",
      "the 128500 setps AND the loss on the train is:0.6135465502738953\n",
      "0.9493\n",
      "the 129000 setps AND the loss on the train is:0.5915918350219727\n",
      "0.9497\n",
      "the 129500 setps AND the loss on the train is:0.5771814584732056\n",
      "0.9501\n",
      "the 130000 setps AND the loss on the train is:0.582857608795166\n",
      "0.9497\n",
      "the 130500 setps AND the loss on the train is:0.6377798318862915\n",
      "0.9501\n",
      "the 131000 setps AND the loss on the train is:0.5274160504341125\n",
      "0.95\n",
      "the 131500 setps AND the loss on the train is:0.6330438256263733\n",
      "0.9503\n",
      "the 132000 setps AND the loss on the train is:0.6286337971687317\n",
      "0.9499\n",
      "the 132500 setps AND the loss on the train is:0.5804641246795654\n",
      "0.951\n",
      "the 133000 setps AND the loss on the train is:0.503516435623169\n",
      "0.9496\n",
      "the 133500 setps AND the loss on the train is:0.535136342048645\n",
      "0.9504\n",
      "the 134000 setps AND the loss on the train is:0.5772362351417542\n",
      "0.95\n",
      "the 134500 setps AND the loss on the train is:0.6727422475814819\n",
      "0.9496\n",
      "the 135000 setps AND the loss on the train is:0.6288414597511292\n",
      "0.9502\n",
      "the 135500 setps AND the loss on the train is:0.551515519618988\n",
      "0.9492\n",
      "the 136000 setps AND the loss on the train is:0.5579895973205566\n",
      "0.9496\n",
      "the 136500 setps AND the loss on the train is:0.5827176570892334\n",
      "0.9498\n",
      "the 137000 setps AND the loss on the train is:0.5474262833595276\n",
      "0.95\n",
      "the 137500 setps AND the loss on the train is:0.5670958161354065\n",
      "0.9504\n",
      "the 138000 setps AND the loss on the train is:0.6341590881347656\n",
      "0.949\n",
      "the 138500 setps AND the loss on the train is:0.5411362648010254\n",
      "0.9504\n",
      "the 139000 setps AND the loss on the train is:0.6405795216560364\n",
      "0.9506\n",
      "the 139500 setps AND the loss on the train is:0.5885788798332214\n",
      "0.9505\n",
      "the 140000 setps AND the loss on the train is:0.7159162759780884\n",
      "0.95\n",
      "the 140500 setps AND the loss on the train is:0.6336597800254822\n",
      "0.9508\n",
      "the 141000 setps AND the loss on the train is:0.5788255333900452\n",
      "0.9511\n",
      "the 141500 setps AND the loss on the train is:0.6219654679298401\n",
      "0.9505\n",
      "the 142000 setps AND the loss on the train is:0.630510687828064\n",
      "0.9503\n",
      "the 142500 setps AND the loss on the train is:0.5162898898124695\n",
      "0.9512\n",
      "the 143000 setps AND the loss on the train is:0.5596141815185547\n",
      "0.9503\n",
      "the 143500 setps AND the loss on the train is:0.6084138751029968\n",
      "0.951\n",
      "the 144000 setps AND the loss on the train is:0.593643307685852\n",
      "0.9503\n",
      "the 144500 setps AND the loss on the train is:0.5836237668991089\n",
      "0.9512\n",
      "the 145000 setps AND the loss on the train is:0.6463298797607422\n",
      "0.9497\n",
      "the 145500 setps AND the loss on the train is:0.6214644908905029\n",
      "0.95\n",
      "the 146000 setps AND the loss on the train is:0.6307386159896851\n",
      "0.9513\n",
      "the 146500 setps AND the loss on the train is:0.5871978998184204\n",
      "0.9501\n",
      "the 147000 setps AND the loss on the train is:0.560224711894989\n",
      "0.9506\n",
      "the 147500 setps AND the loss on the train is:0.602936863899231\n",
      "0.9513\n",
      "the 148000 setps AND the loss on the train is:0.6058194637298584\n",
      "0.9514\n",
      "the 148500 setps AND the loss on the train is:0.5699847340583801\n",
      "0.9513\n",
      "the 149000 setps AND the loss on the train is:0.5916284322738647\n",
      "0.9516\n",
      "the 149500 setps AND the loss on the train is:0.5703020691871643\n",
      "0.9506\n",
      "the 150000 setps AND the loss on the train is:0.7368932962417603\n",
      "0.9509\n",
      "the 150500 setps AND the loss on the train is:0.5698522925376892\n",
      "0.9503\n",
      "the 151000 setps AND the loss on the train is:0.6912720799446106\n",
      "0.9498\n",
      "the 151500 setps AND the loss on the train is:0.5457735061645508\n",
      "0.9508\n",
      "the 152000 setps AND the loss on the train is:0.6323582530021667\n",
      "0.951\n",
      "the 152500 setps AND the loss on the train is:0.5469493865966797\n",
      "0.9503\n",
      "the 153000 setps AND the loss on the train is:0.5894474983215332\n",
      "0.9519\n",
      "the 153500 setps AND the loss on the train is:0.6152339577674866\n",
      "0.9513\n",
      "the 154000 setps AND the loss on the train is:0.5605401992797852\n",
      "0.9515\n",
      "the 154500 setps AND the loss on the train is:0.5449569821357727\n",
      "0.9497\n",
      "the 155000 setps AND the loss on the train is:0.5719364881515503\n",
      "0.9495\n",
      "the 155500 setps AND the loss on the train is:0.5500657558441162\n",
      "0.9495\n",
      "the 156000 setps AND the loss on the train is:0.5360956788063049\n",
      "0.9511\n",
      "the 156500 setps AND the loss on the train is:0.6011935472488403\n",
      "0.9505\n",
      "the 157000 setps AND the loss on the train is:0.5826913118362427\n",
      "0.9511\n",
      "the 157500 setps AND the loss on the train is:0.6543654203414917\n",
      "0.9513\n",
      "the 158000 setps AND the loss on the train is:0.6821554899215698\n",
      "0.9506\n",
      "the 158500 setps AND the loss on the train is:0.5534387826919556\n",
      "0.9513\n",
      "the 159000 setps AND the loss on the train is:0.6410455703735352\n",
      "0.9499\n",
      "the 159500 setps AND the loss on the train is:0.5271193385124207\n",
      "0.9508\n",
      "the 160000 setps AND the loss on the train is:0.6148940920829773\n",
      "0.9514\n",
      "the 160500 setps AND the loss on the train is:0.6208056211471558\n",
      "0.9506\n",
      "the 161000 setps AND the loss on the train is:0.5702546238899231\n",
      "0.9506\n",
      "the 161500 setps AND the loss on the train is:0.6002529263496399\n",
      "0.9516\n",
      "the 162000 setps AND the loss on the train is:0.5832567811012268\n",
      "0.9505\n",
      "the 162500 setps AND the loss on the train is:0.5387011766433716\n",
      "0.9504\n",
      "the 163000 setps AND the loss on the train is:0.5774745941162109\n",
      "0.9498\n",
      "the 163500 setps AND the loss on the train is:0.6708139181137085\n",
      "0.9495\n",
      "the 164000 setps AND the loss on the train is:0.6177279949188232\n",
      "0.9505\n",
      "the 164500 setps AND the loss on the train is:0.6123124361038208\n",
      "0.951\n",
      "the 165000 setps AND the loss on the train is:0.7233675122261047\n",
      "0.9512\n",
      "the 165500 setps AND the loss on the train is:0.5711573958396912\n",
      "0.9509\n",
      "the 166000 setps AND the loss on the train is:0.6591572761535645\n",
      "0.9517\n",
      "the 166500 setps AND the loss on the train is:0.5472582578659058\n",
      "0.9511\n",
      "the 167000 setps AND the loss on the train is:0.594146728515625\n",
      "0.9513\n",
      "the 167500 setps AND the loss on the train is:0.547265350818634\n",
      "0.9511\n",
      "the 168000 setps AND the loss on the train is:0.5687976479530334\n",
      "0.9507\n",
      "the 168500 setps AND the loss on the train is:0.6235727667808533\n",
      "0.9512\n",
      "the 169000 setps AND the loss on the train is:0.6187822818756104\n",
      "0.9506\n",
      "the 169500 setps AND the loss on the train is:0.5920547842979431\n",
      "0.9516\n",
      "the 170000 setps AND the loss on the train is:0.6872186660766602\n",
      "0.9507\n",
      "the 170500 setps AND the loss on the train is:0.5888354182243347\n",
      "0.9504\n",
      "the 171000 setps AND the loss on the train is:0.5443968772888184\n",
      "0.9511\n",
      "the 171500 setps AND the loss on the train is:0.6099672317504883\n",
      "0.9515\n",
      "the 172000 setps AND the loss on the train is:0.6343764066696167\n",
      "0.9515\n",
      "the 172500 setps AND the loss on the train is:0.6514564752578735\n",
      "0.9514\n",
      "the 173000 setps AND the loss on the train is:0.5579068660736084\n",
      "0.9503\n",
      "the 173500 setps AND the loss on the train is:0.6134957671165466\n",
      "0.9499\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 174000 setps AND the loss on the train is:0.560846209526062\n",
      "0.9511\n",
      "the 174500 setps AND the loss on the train is:0.627009391784668\n",
      "0.9514\n",
      "the 175000 setps AND the loss on the train is:0.6046492457389832\n",
      "0.9493\n",
      "the 175500 setps AND the loss on the train is:0.5565962791442871\n",
      "0.9513\n",
      "the 176000 setps AND the loss on the train is:0.5023699998855591\n",
      "0.9503\n",
      "the 176500 setps AND the loss on the train is:0.5810648202896118\n",
      "0.9513\n",
      "the 177000 setps AND the loss on the train is:0.5653353333473206\n",
      "0.9506\n",
      "the 177500 setps AND the loss on the train is:0.6634454727172852\n",
      "0.9508\n",
      "the 178000 setps AND the loss on the train is:0.6158801913261414\n",
      "0.951\n",
      "the 178500 setps AND the loss on the train is:0.621514081954956\n",
      "0.95\n",
      "the 179000 setps AND the loss on the train is:0.5536909103393555\n",
      "0.9511\n",
      "the 179500 setps AND the loss on the train is:0.6117153763771057\n",
      "0.9508\n",
      "the 180000 setps AND the loss on the train is:0.5404148697853088\n",
      "0.9514\n",
      "the 180500 setps AND the loss on the train is:0.5676209926605225\n",
      "0.95\n",
      "the 181000 setps AND the loss on the train is:0.6242901086807251\n",
      "0.9504\n",
      "the 181500 setps AND the loss on the train is:0.555338442325592\n",
      "0.9513\n",
      "the 182000 setps AND the loss on the train is:0.5507082939147949\n",
      "0.9504\n",
      "the 182500 setps AND the loss on the train is:0.5938044190406799\n",
      "0.9504\n",
      "the 183000 setps AND the loss on the train is:0.6136001348495483\n",
      "0.9503\n",
      "the 183500 setps AND the loss on the train is:0.6294493675231934\n",
      "0.9505\n",
      "the 184000 setps AND the loss on the train is:0.643388032913208\n",
      "0.9512\n",
      "the 184500 setps AND the loss on the train is:0.5832328796386719\n",
      "0.9505\n",
      "the 185000 setps AND the loss on the train is:0.5753597617149353\n",
      "0.9508\n",
      "the 185500 setps AND the loss on the train is:0.6575043201446533\n",
      "0.9525\n",
      "the 186000 setps AND the loss on the train is:0.5900372266769409\n",
      "0.9513\n",
      "the 186500 setps AND the loss on the train is:0.5926938056945801\n",
      "0.9512\n",
      "the 187000 setps AND the loss on the train is:0.5791411995887756\n",
      "0.9502\n",
      "the 187500 setps AND the loss on the train is:0.5897314548492432\n",
      "0.9517\n",
      "the 188000 setps AND the loss on the train is:0.6132237911224365\n",
      "0.9512\n",
      "the 188500 setps AND the loss on the train is:0.5629531145095825\n",
      "0.9506\n",
      "the 189000 setps AND the loss on the train is:0.6288868188858032\n",
      "0.9515\n",
      "the 189500 setps AND the loss on the train is:0.6935365200042725\n",
      "0.9519\n",
      "the 190000 setps AND the loss on the train is:0.6753643751144409\n",
      "0.9511\n",
      "the 190500 setps AND the loss on the train is:0.6313828229904175\n",
      "0.9508\n",
      "the 191000 setps AND the loss on the train is:0.5721749663352966\n",
      "0.9505\n",
      "the 191500 setps AND the loss on the train is:0.5702189207077026\n",
      "0.9512\n",
      "the 192000 setps AND the loss on the train is:0.5553743243217468\n",
      "0.951\n",
      "the 192500 setps AND the loss on the train is:0.5422605872154236\n",
      "0.9508\n",
      "the 193000 setps AND the loss on the train is:0.5598265528678894\n",
      "0.9513\n",
      "the 193500 setps AND the loss on the train is:0.6342547535896301\n",
      "0.9509\n",
      "the 194000 setps AND the loss on the train is:0.6136394739151001\n",
      "0.9518\n",
      "the 194500 setps AND the loss on the train is:0.6188853979110718\n",
      "0.9492\n",
      "the 195000 setps AND the loss on the train is:0.5863955020904541\n",
      "0.9504\n",
      "the 195500 setps AND the loss on the train is:0.6283412575721741\n",
      "0.9503\n",
      "the 196000 setps AND the loss on the train is:0.5684847235679626\n",
      "0.9517\n",
      "the 196500 setps AND the loss on the train is:0.6405232548713684\n",
      "0.9507\n",
      "the 197000 setps AND the loss on the train is:0.5349525809288025\n",
      "0.9504\n",
      "the 197500 setps AND the loss on the train is:0.644213080406189\n",
      "0.9505\n",
      "the 198000 setps AND the loss on the train is:0.6369591951370239\n",
      "0.951\n",
      "the 198500 setps AND the loss on the train is:0.5769429206848145\n",
      "0.9509\n",
      "the 199000 setps AND the loss on the train is:0.5878332853317261\n",
      "0.9516\n",
      "the 199500 setps AND the loss on the train is:0.6358913779258728\n",
      "0.9515\n",
      "the 200000 setps AND the loss on the train is:0.5571305751800537\n",
      "0.9504\n",
      "the 200500 setps AND the loss on the train is:0.6199648380279541\n",
      "0.9515\n",
      "the 201000 setps AND the loss on the train is:0.578241229057312\n",
      "0.9495\n",
      "the 201500 setps AND the loss on the train is:0.6042966246604919\n",
      "0.9513\n",
      "the 202000 setps AND the loss on the train is:0.5087060928344727\n",
      "0.9514\n",
      "the 202500 setps AND the loss on the train is:0.5627726316452026\n",
      "0.9514\n",
      "the 203000 setps AND the loss on the train is:0.5654563903808594\n",
      "0.9507\n",
      "the 203500 setps AND the loss on the train is:0.6160455942153931\n",
      "0.9512\n",
      "the 204000 setps AND the loss on the train is:0.5317381620407104\n",
      "0.951\n",
      "the 204500 setps AND the loss on the train is:0.6397638320922852\n",
      "0.9513\n",
      "the 205000 setps AND the loss on the train is:0.6344925165176392\n",
      "0.9513\n",
      "the 205500 setps AND the loss on the train is:0.6762768030166626\n",
      "0.95\n",
      "the 206000 setps AND the loss on the train is:0.5757848024368286\n",
      "0.9512\n",
      "the 206500 setps AND the loss on the train is:0.5983474850654602\n",
      "0.9515\n",
      "the 207000 setps AND the loss on the train is:0.5802907347679138\n",
      "0.9517\n",
      "the 207500 setps AND the loss on the train is:0.5566091537475586\n",
      "0.951\n",
      "the 208000 setps AND the loss on the train is:0.6047844290733337\n",
      "0.9512\n",
      "the 208500 setps AND the loss on the train is:0.7168399095535278\n",
      "0.951\n",
      "the 209000 setps AND the loss on the train is:0.5356400609016418\n",
      "0.9507\n",
      "the 209500 setps AND the loss on the train is:0.5888368487358093\n",
      "0.9512\n",
      "the 210000 setps AND the loss on the train is:0.5819816589355469\n",
      "0.9506\n",
      "the 210500 setps AND the loss on the train is:0.6707145571708679\n",
      "0.9514\n",
      "the 211000 setps AND the loss on the train is:0.5572245121002197\n",
      "0.9502\n",
      "the 211500 setps AND the loss on the train is:0.6085280776023865\n",
      "0.9518\n",
      "the 212000 setps AND the loss on the train is:0.6132833361625671\n",
      "0.9506\n",
      "the 212500 setps AND the loss on the train is:0.5927466750144958\n",
      "0.9514\n",
      "the 213000 setps AND the loss on the train is:0.6154646873474121\n",
      "0.9514\n",
      "the 213500 setps AND the loss on the train is:0.641099214553833\n",
      "0.9515\n",
      "the 214000 setps AND the loss on the train is:0.6343327760696411\n",
      "0.9509\n",
      "the 214500 setps AND the loss on the train is:0.6188730001449585\n",
      "0.9502\n",
      "the 215000 setps AND the loss on the train is:0.6050220727920532\n",
      "0.9511\n",
      "the 215500 setps AND the loss on the train is:0.5592952966690063\n",
      "0.9501\n",
      "the 216000 setps AND the loss on the train is:0.6785365343093872\n",
      "0.9516\n",
      "the 216500 setps AND the loss on the train is:0.5220305919647217\n",
      "0.9516\n",
      "the 217000 setps AND the loss on the train is:0.569267988204956\n",
      "0.95\n",
      "the 217500 setps AND the loss on the train is:0.6108002066612244\n",
      "0.9513\n",
      "the 218000 setps AND the loss on the train is:0.6466246843338013\n",
      "0.9514\n",
      "the 218500 setps AND the loss on the train is:0.6151731014251709\n",
      "0.9511\n",
      "the 219000 setps AND the loss on the train is:0.5067867040634155\n",
      "0.9511\n",
      "the 219500 setps AND the loss on the train is:0.5218204259872437\n",
      "0.9508\n",
      "the 220000 setps AND the loss on the train is:0.7051527500152588\n",
      "0.9502\n",
      "the 220500 setps AND the loss on the train is:0.5129737854003906\n",
      "0.9522\n",
      "the 221000 setps AND the loss on the train is:0.6242367029190063\n",
      "0.9502\n",
      "the 221500 setps AND the loss on the train is:0.6184799075126648\n",
      "0.9516\n",
      "the 222000 setps AND the loss on the train is:0.6064919233322144\n",
      "0.9508\n",
      "the 222500 setps AND the loss on the train is:0.5055146813392639\n",
      "0.9511\n",
      "the 223000 setps AND the loss on the train is:0.5860073566436768\n",
      "0.9517\n",
      "the 223500 setps AND the loss on the train is:0.5972751975059509\n",
      "0.9502\n",
      "the 224000 setps AND the loss on the train is:0.5399141311645508\n",
      "0.9507\n",
      "the 224500 setps AND the loss on the train is:0.5749289989471436\n",
      "0.9506\n",
      "the 225000 setps AND the loss on the train is:0.5496583580970764\n",
      "0.9516\n",
      "the 225500 setps AND the loss on the train is:0.6047257781028748\n",
      "0.9507\n",
      "the 226000 setps AND the loss on the train is:0.5537393093109131\n",
      "0.9502\n",
      "the 226500 setps AND the loss on the train is:0.6302799582481384\n",
      "0.9514\n",
      "the 227000 setps AND the loss on the train is:0.5627470016479492\n",
      "0.9515\n",
      "the 227500 setps AND the loss on the train is:0.5979952216148376\n",
      "0.95\n",
      "the 228000 setps AND the loss on the train is:0.5335497856140137\n",
      "0.9512\n",
      "the 228500 setps AND the loss on the train is:0.5533958673477173\n",
      "0.9505\n",
      "the 229000 setps AND the loss on the train is:0.5692998766899109\n",
      "0.951\n",
      "the 229500 setps AND the loss on the train is:0.64360511302948\n",
      "0.9513\n",
      "the 230000 setps AND the loss on the train is:0.5710455775260925\n",
      "0.9513\n",
      "the 230500 setps AND the loss on the train is:0.5669878125190735\n",
      "0.9502\n",
      "the 231000 setps AND the loss on the train is:0.560992956161499\n",
      "0.9501\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 231500 setps AND the loss on the train is:0.6069222688674927\n",
      "0.9511\n",
      "the 232000 setps AND the loss on the train is:0.6008221507072449\n",
      "0.9515\n",
      "the 232500 setps AND the loss on the train is:0.5503175854682922\n",
      "0.9521\n",
      "the 233000 setps AND the loss on the train is:0.624882698059082\n",
      "0.9521\n",
      "the 233500 setps AND the loss on the train is:0.5913274884223938\n",
      "0.9509\n",
      "the 234000 setps AND the loss on the train is:0.5862622857093811\n",
      "0.9511\n",
      "the 234500 setps AND the loss on the train is:0.7386279106140137\n",
      "0.9516\n",
      "the 235000 setps AND the loss on the train is:0.5443413257598877\n",
      "0.9518\n",
      "the 235500 setps AND the loss on the train is:0.5710834264755249\n",
      "0.9512\n",
      "the 236000 setps AND the loss on the train is:0.5142534375190735\n",
      "0.9515\n",
      "the 236500 setps AND the loss on the train is:0.5958956480026245\n",
      "0.9509\n",
      "the 237000 setps AND the loss on the train is:0.5871559977531433\n",
      "0.9516\n",
      "the 237500 setps AND the loss on the train is:0.6070350408554077\n",
      "0.9507\n",
      "the 238000 setps AND the loss on the train is:0.5546379089355469\n",
      "0.9512\n",
      "the 238500 setps AND the loss on the train is:0.5753689408302307\n",
      "0.9513\n",
      "the 239000 setps AND the loss on the train is:0.5687384605407715\n",
      "0.9507\n",
      "the 239500 setps AND the loss on the train is:0.5514571666717529\n",
      "0.9504\n",
      "the 240000 setps AND the loss on the train is:0.6039159893989563\n",
      "0.9513\n",
      "the 240500 setps AND the loss on the train is:0.563512921333313\n",
      "0.9511\n",
      "the 241000 setps AND the loss on the train is:0.6302046775817871\n",
      "0.9507\n",
      "the 241500 setps AND the loss on the train is:0.5660672187805176\n",
      "0.9513\n",
      "the 242000 setps AND the loss on the train is:0.6007245779037476\n",
      "0.9506\n",
      "the 242500 setps AND the loss on the train is:0.5455514192581177\n",
      "0.9506\n",
      "the 243000 setps AND the loss on the train is:0.49897366762161255\n",
      "0.9509\n",
      "the 243500 setps AND the loss on the train is:0.6567087173461914\n",
      "0.9517\n",
      "the 244000 setps AND the loss on the train is:0.6459423303604126\n",
      "0.9504\n",
      "the 244500 setps AND the loss on the train is:0.5888572931289673\n",
      "0.9508\n",
      "the 245000 setps AND the loss on the train is:0.6845822334289551\n",
      "0.9501\n",
      "the 245500 setps AND the loss on the train is:0.5171209573745728\n",
      "0.9505\n",
      "the 246000 setps AND the loss on the train is:0.4605128765106201\n",
      "0.9512\n",
      "the 246500 setps AND the loss on the train is:0.5308945178985596\n",
      "0.9504\n",
      "the 247000 setps AND the loss on the train is:0.6101647615432739\n",
      "0.9502\n",
      "the 247500 setps AND the loss on the train is:0.5689918398857117\n",
      "0.95\n",
      "the 248000 setps AND the loss on the train is:0.5485700368881226\n",
      "0.9513\n",
      "the 248500 setps AND the loss on the train is:0.5736218094825745\n",
      "0.9501\n",
      "the 249000 setps AND the loss on the train is:0.7695608139038086\n",
      "0.9508\n",
      "the 249500 setps AND the loss on the train is:0.5742093920707703\n",
      "0.9506\n",
      "the 250000 setps AND the loss on the train is:0.5611103177070618\n",
      "0.9515\n",
      "the 250500 setps AND the loss on the train is:0.5563334226608276\n",
      "0.9517\n",
      "the 251000 setps AND the loss on the train is:0.5554412007331848\n",
      "0.95\n",
      "the 251500 setps AND the loss on the train is:0.627231240272522\n",
      "0.9504\n",
      "the 252000 setps AND the loss on the train is:0.6091296672821045\n",
      "0.9516\n",
      "the 252500 setps AND the loss on the train is:0.5597047209739685\n",
      "0.9508\n",
      "the 253000 setps AND the loss on the train is:0.5533897876739502\n",
      "0.9494\n",
      "the 253500 setps AND the loss on the train is:0.5966137647628784\n",
      "0.9506\n",
      "the 254000 setps AND the loss on the train is:0.6779752969741821\n",
      "0.9507\n",
      "the 254500 setps AND the loss on the train is:0.5542982220649719\n",
      "0.9511\n",
      "the 255000 setps AND the loss on the train is:0.6339023113250732\n",
      "0.9506\n",
      "the 255500 setps AND the loss on the train is:0.584891676902771\n",
      "0.9507\n",
      "the 256000 setps AND the loss on the train is:0.5464355945587158\n",
      "0.9501\n",
      "the 256500 setps AND the loss on the train is:0.5538380146026611\n",
      "0.9506\n",
      "the 257000 setps AND the loss on the train is:0.5520696043968201\n",
      "0.9512\n",
      "the 257500 setps AND the loss on the train is:0.5671701431274414\n",
      "0.9499\n",
      "the 258000 setps AND the loss on the train is:0.7012183666229248\n",
      "0.9516\n",
      "the 258500 setps AND the loss on the train is:0.5767021179199219\n",
      "0.9516\n",
      "the 259000 setps AND the loss on the train is:0.6223516464233398\n",
      "0.9508\n",
      "the 259500 setps AND the loss on the train is:0.6710867285728455\n",
      "0.9503\n",
      "the 260000 setps AND the loss on the train is:0.6041251420974731\n",
      "0.9516\n",
      "the 260500 setps AND the loss on the train is:0.6999445557594299\n",
      "0.9513\n",
      "the 261000 setps AND the loss on the train is:0.5953753590583801\n",
      "0.9512\n",
      "the 261500 setps AND the loss on the train is:0.6188147068023682\n",
      "0.9507\n",
      "the 262000 setps AND the loss on the train is:0.5119668245315552\n",
      "0.9516\n",
      "the 262500 setps AND the loss on the train is:0.6173939108848572\n",
      "0.9512\n",
      "the 263000 setps AND the loss on the train is:0.6007457971572876\n",
      "0.9506\n",
      "the 263500 setps AND the loss on the train is:0.6056057214736938\n",
      "0.9506\n",
      "the 264000 setps AND the loss on the train is:0.6405479907989502\n",
      "0.9517\n",
      "the 264500 setps AND the loss on the train is:0.681698203086853\n",
      "0.9507\n",
      "the 265000 setps AND the loss on the train is:0.6330821514129639\n",
      "0.9513\n",
      "the 265500 setps AND the loss on the train is:0.5667749643325806\n",
      "0.951\n",
      "the 266000 setps AND the loss on the train is:0.6234376430511475\n",
      "0.9512\n",
      "the 266500 setps AND the loss on the train is:0.5337300896644592\n",
      "0.9509\n",
      "the 267000 setps AND the loss on the train is:0.5790560245513916\n",
      "0.9512\n",
      "the 267500 setps AND the loss on the train is:0.635327935218811\n",
      "0.952\n",
      "the 268000 setps AND the loss on the train is:0.7176522016525269\n",
      "0.9507\n",
      "the 268500 setps AND the loss on the train is:0.6101653575897217\n",
      "0.9503\n",
      "the 269000 setps AND the loss on the train is:0.5954367518424988\n",
      "0.9506\n",
      "the 269500 setps AND the loss on the train is:0.5893651843070984\n",
      "0.9508\n",
      "the 270000 setps AND the loss on the train is:0.6352113485336304\n",
      "0.9508\n",
      "the 270500 setps AND the loss on the train is:0.5627583265304565\n",
      "0.9525\n",
      "the 271000 setps AND the loss on the train is:0.5850405097007751\n",
      "0.9505\n",
      "the 271500 setps AND the loss on the train is:0.5956276655197144\n",
      "0.951\n",
      "the 272000 setps AND the loss on the train is:0.5480995774269104\n",
      "0.9495\n",
      "the 272500 setps AND the loss on the train is:0.6388874053955078\n",
      "0.9513\n",
      "the 273000 setps AND the loss on the train is:0.5528252124786377\n",
      "0.951\n",
      "the 273500 setps AND the loss on the train is:0.584488034248352\n",
      "0.9505\n",
      "the 274000 setps AND the loss on the train is:0.6209103465080261\n",
      "0.9508\n",
      "the 274500 setps AND the loss on the train is:0.5659753084182739\n",
      "0.9512\n",
      "the 275000 setps AND the loss on the train is:0.5882169604301453\n",
      "0.951\n",
      "the 275500 setps AND the loss on the train is:0.6388224363327026\n",
      "0.9511\n",
      "the 276000 setps AND the loss on the train is:0.543120801448822\n",
      "0.9506\n",
      "the 276500 setps AND the loss on the train is:0.698654294013977\n",
      "0.9499\n",
      "the 277000 setps AND the loss on the train is:0.625227153301239\n",
      "0.9506\n",
      "the 277500 setps AND the loss on the train is:0.6722566485404968\n",
      "0.9508\n",
      "the 278000 setps AND the loss on the train is:0.6011330485343933\n",
      "0.95\n",
      "the 278500 setps AND the loss on the train is:0.5394638776779175\n",
      "0.9518\n",
      "the 279000 setps AND the loss on the train is:0.6106479167938232\n",
      "0.951\n",
      "the 279500 setps AND the loss on the train is:0.5170808434486389\n",
      "0.9509\n",
      "the 280000 setps AND the loss on the train is:0.5938961505889893\n",
      "0.9514\n",
      "the 280500 setps AND the loss on the train is:0.6526567935943604\n",
      "0.9507\n",
      "the 281000 setps AND the loss on the train is:0.695725679397583\n",
      "0.951\n",
      "the 281500 setps AND the loss on the train is:0.7136292457580566\n",
      "0.9516\n",
      "the 282000 setps AND the loss on the train is:0.5547889471054077\n",
      "0.9511\n",
      "the 282500 setps AND the loss on the train is:0.5626296997070312\n",
      "0.9505\n",
      "the 283000 setps AND the loss on the train is:0.6233582496643066\n",
      "0.9518\n",
      "the 283500 setps AND the loss on the train is:0.594083309173584\n",
      "0.9502\n",
      "the 284000 setps AND the loss on the train is:0.6499096155166626\n",
      "0.9495\n",
      "the 284500 setps AND the loss on the train is:0.6114262342453003\n",
      "0.9507\n",
      "the 285000 setps AND the loss on the train is:0.6291965246200562\n",
      "0.9514\n",
      "the 285500 setps AND the loss on the train is:0.5891292691230774\n",
      "0.9502\n",
      "the 286000 setps AND the loss on the train is:0.6159713268280029\n",
      "0.9518\n",
      "the 286500 setps AND the loss on the train is:0.5691448450088501\n",
      "0.9513\n",
      "the 287000 setps AND the loss on the train is:0.6063210964202881\n",
      "0.951\n",
      "the 287500 setps AND the loss on the train is:0.600866973400116\n",
      "0.951\n",
      "the 288000 setps AND the loss on the train is:0.5689687728881836\n",
      "0.9508\n",
      "the 288500 setps AND the loss on the train is:0.6333761215209961\n",
      "0.9501\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 289000 setps AND the loss on the train is:0.5337846279144287\n",
      "0.951\n",
      "the 289500 setps AND the loss on the train is:0.5542340278625488\n",
      "0.9505\n",
      "the 290000 setps AND the loss on the train is:0.6124554872512817\n",
      "0.9514\n",
      "the 290500 setps AND the loss on the train is:0.6019608974456787\n",
      "0.9508\n",
      "the 291000 setps AND the loss on the train is:0.49808329343795776\n",
      "0.9504\n",
      "the 291500 setps AND the loss on the train is:0.6020593047142029\n",
      "0.9508\n",
      "the 292000 setps AND the loss on the train is:0.5186885595321655\n",
      "0.9506\n",
      "the 292500 setps AND the loss on the train is:0.5934100151062012\n",
      "0.9506\n",
      "the 293000 setps AND the loss on the train is:0.6071712374687195\n",
      "0.9505\n",
      "the 293500 setps AND the loss on the train is:0.5188174247741699\n",
      "0.9517\n",
      "the 294000 setps AND the loss on the train is:0.5389798879623413\n",
      "0.9513\n",
      "the 294500 setps AND the loss on the train is:0.5664469003677368\n",
      "0.95\n",
      "the 295000 setps AND the loss on the train is:0.664746105670929\n",
      "0.951\n",
      "the 295500 setps AND the loss on the train is:0.650032639503479\n",
      "0.9514\n",
      "the 296000 setps AND the loss on the train is:0.5645198225975037\n",
      "0.9519\n",
      "the 296500 setps AND the loss on the train is:0.6449922323226929\n",
      "0.9503\n",
      "the 297000 setps AND the loss on the train is:0.7322554588317871\n",
      "0.9511\n",
      "the 297500 setps AND the loss on the train is:0.5599951148033142\n",
      "0.9518\n",
      "the 298000 setps AND the loss on the train is:0.555702805519104\n",
      "0.9517\n",
      "the 298500 setps AND the loss on the train is:0.6155494451522827\n",
      "0.95\n",
      "the 299000 setps AND the loss on the train is:0.5856325626373291\n",
      "0.9501\n",
      "the 299500 setps AND the loss on the train is:0.5734575986862183\n",
      "0.9507\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,50],0.01)\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=get_weight([50,50],0.01)\n",
    "b2=tf.Variable(tf.random_normal([50]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.relu(logits2)\n",
    "\n",
    "w3=get_weight([50,10],0.01)\n",
    "b3=tf.Variable(tf.random_normal([10]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits3))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits3,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(300000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4.2 在3.4.1基础上将学习率设定为0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:3264.735595703125\n",
      "0.0702\n",
      "the 500 setps AND the loss on the train is:2844.91748046875\n",
      "0.5363\n",
      "the 1000 setps AND the loss on the train is:2799.11376953125\n",
      "0.6155\n",
      "the 1500 setps AND the loss on the train is:2765.55712890625\n",
      "0.6521\n",
      "the 2000 setps AND the loss on the train is:2740.475341796875\n",
      "0.6697\n",
      "the 2500 setps AND the loss on the train is:2712.1298828125\n",
      "0.6802\n",
      "the 3000 setps AND the loss on the train is:2679.78076171875\n",
      "0.6888\n",
      "the 3500 setps AND the loss on the train is:2652.8095703125\n",
      "0.6961\n",
      "the 4000 setps AND the loss on the train is:2624.491455078125\n",
      "0.6993\n",
      "the 4500 setps AND the loss on the train is:2597.300048828125\n",
      "0.7022\n",
      "the 5000 setps AND the loss on the train is:2573.06103515625\n",
      "0.7004\n",
      "the 5500 setps AND the loss on the train is:2545.486328125\n",
      "0.695\n",
      "the 6000 setps AND the loss on the train is:2518.953369140625\n",
      "0.7044\n",
      "the 6500 setps AND the loss on the train is:2493.22314453125\n",
      "0.7023\n",
      "the 7000 setps AND the loss on the train is:2467.75390625\n",
      "0.7045\n",
      "the 7500 setps AND the loss on the train is:2443.11572265625\n",
      "0.7021\n",
      "the 8000 setps AND the loss on the train is:2417.74267578125\n",
      "0.6998\n",
      "the 8500 setps AND the loss on the train is:2393.1337890625\n",
      "0.7115\n",
      "the 9000 setps AND the loss on the train is:2368.3857421875\n",
      "0.678\n",
      "the 9500 setps AND the loss on the train is:2343.820556640625\n",
      "0.6941\n",
      "the 10000 setps AND the loss on the train is:2320.784912109375\n",
      "0.6735\n",
      "the 10500 setps AND the loss on the train is:2296.458251953125\n",
      "0.7337\n",
      "the 11000 setps AND the loss on the train is:2273.10791015625\n",
      "0.6855\n",
      "the 11500 setps AND the loss on the train is:2249.372314453125\n",
      "0.731\n",
      "the 12000 setps AND the loss on the train is:2226.763427734375\n",
      "0.7364\n",
      "the 12500 setps AND the loss on the train is:2203.72314453125\n",
      "0.726\n",
      "the 13000 setps AND the loss on the train is:2180.756591796875\n",
      "0.7184\n",
      "the 13500 setps AND the loss on the train is:2158.325927734375\n",
      "0.7354\n",
      "the 14000 setps AND the loss on the train is:2136.44384765625\n",
      "0.7322\n",
      "the 14500 setps AND the loss on the train is:2114.245361328125\n",
      "0.7201\n",
      "the 15000 setps AND the loss on the train is:2092.485107421875\n",
      "0.7251\n",
      "the 15500 setps AND the loss on the train is:2071.0693359375\n",
      "0.7078\n",
      "the 16000 setps AND the loss on the train is:2049.867431640625\n",
      "0.72\n",
      "the 16500 setps AND the loss on the train is:2028.377197265625\n",
      "0.7267\n",
      "the 17000 setps AND the loss on the train is:2007.3858642578125\n",
      "0.7612\n",
      "the 17500 setps AND the loss on the train is:1986.5472412109375\n",
      "0.7533\n",
      "the 18000 setps AND the loss on the train is:1965.7755126953125\n",
      "0.7231\n",
      "the 18500 setps AND the loss on the train is:1945.79833984375\n",
      "0.7552\n",
      "the 19000 setps AND the loss on the train is:1925.4805908203125\n",
      "0.7385\n",
      "the 19500 setps AND the loss on the train is:1905.2735595703125\n",
      "0.772\n",
      "the 20000 setps AND the loss on the train is:1885.695068359375\n",
      "0.7549\n",
      "the 20500 setps AND the loss on the train is:1866.15185546875\n",
      "0.7653\n",
      "the 21000 setps AND the loss on the train is:1846.45751953125\n",
      "0.7542\n",
      "the 21500 setps AND the loss on the train is:1827.257568359375\n",
      "0.7672\n",
      "the 22000 setps AND the loss on the train is:1808.0235595703125\n",
      "0.7387\n",
      "the 22500 setps AND the loss on the train is:1789.3326416015625\n",
      "0.7836\n",
      "the 23000 setps AND the loss on the train is:1770.7318115234375\n",
      "0.7693\n",
      "the 23500 setps AND the loss on the train is:1752.0406494140625\n",
      "0.783\n",
      "the 24000 setps AND the loss on the train is:1733.5609130859375\n",
      "0.7658\n",
      "the 24500 setps AND the loss on the train is:1715.6475830078125\n",
      "0.7681\n",
      "the 25000 setps AND the loss on the train is:1697.580810546875\n",
      "0.7844\n",
      "the 25500 setps AND the loss on the train is:1679.65673828125\n",
      "0.7626\n",
      "the 26000 setps AND the loss on the train is:1661.984375\n",
      "0.7873\n",
      "the 26500 setps AND the loss on the train is:1644.3436279296875\n",
      "0.7795\n",
      "the 27000 setps AND the loss on the train is:1627.1510009765625\n",
      "0.7753\n",
      "the 27500 setps AND the loss on the train is:1610.0244140625\n",
      "0.7912\n",
      "the 28000 setps AND the loss on the train is:1593.173583984375\n",
      "0.7883\n",
      "the 28500 setps AND the loss on the train is:1575.9932861328125\n",
      "0.7902\n",
      "the 29000 setps AND the loss on the train is:1559.5126953125\n",
      "0.7953\n",
      "the 29500 setps AND the loss on the train is:1542.8568115234375\n",
      "0.7874\n",
      "the 30000 setps AND the loss on the train is:1526.5809326171875\n",
      "0.7886\n",
      "the 30500 setps AND the loss on the train is:1510.6114501953125\n",
      "0.7941\n",
      "the 31000 setps AND the loss on the train is:1494.473388671875\n",
      "0.7982\n",
      "the 31500 setps AND the loss on the train is:1478.7957763671875\n",
      "0.7975\n",
      "the 32000 setps AND the loss on the train is:1462.9847412109375\n",
      "0.7959\n",
      "the 32500 setps AND the loss on the train is:1447.32080078125\n",
      "0.7932\n",
      "the 33000 setps AND the loss on the train is:1432.02978515625\n",
      "0.795\n",
      "the 33500 setps AND the loss on the train is:1416.8240966796875\n",
      "0.7965\n",
      "the 34000 setps AND the loss on the train is:1401.6453857421875\n",
      "0.7925\n",
      "the 34500 setps AND the loss on the train is:1386.6134033203125\n",
      "0.7921\n",
      "the 35000 setps AND the loss on the train is:1371.941650390625\n",
      "0.8017\n",
      "the 35500 setps AND the loss on the train is:1357.1212158203125\n",
      "0.7952\n",
      "the 36000 setps AND the loss on the train is:1342.934326171875\n",
      "0.7958\n",
      "the 36500 setps AND the loss on the train is:1328.6854248046875\n",
      "0.804\n",
      "the 37000 setps AND the loss on the train is:1314.1417236328125\n",
      "0.8048\n",
      "the 37500 setps AND the loss on the train is:1299.935302734375\n",
      "0.8072\n",
      "the 38000 setps AND the loss on the train is:1286.1168212890625\n",
      "0.8053\n",
      "the 38500 setps AND the loss on the train is:1272.32568359375\n",
      "0.8026\n",
      "the 39000 setps AND the loss on the train is:1258.68359375\n",
      "0.8134\n",
      "the 39500 setps AND the loss on the train is:1245.0968017578125\n",
      "0.805\n",
      "the 40000 setps AND the loss on the train is:1231.57666015625\n",
      "0.814\n",
      "the 40500 setps AND the loss on the train is:1218.41455078125\n",
      "0.813\n",
      "the 41000 setps AND the loss on the train is:1205.5361328125\n",
      "0.8136\n",
      "the 41500 setps AND the loss on the train is:1192.2757568359375\n",
      "0.8116\n",
      "the 42000 setps AND the loss on the train is:1179.4625244140625\n",
      "0.8107\n",
      "the 42500 setps AND the loss on the train is:1166.8651123046875\n",
      "0.8174\n",
      "the 43000 setps AND the loss on the train is:1154.232421875\n",
      "0.8126\n",
      "the 43500 setps AND the loss on the train is:1141.771240234375\n",
      "0.8202\n",
      "the 44000 setps AND the loss on the train is:1129.37548828125\n",
      "0.8207\n",
      "the 44500 setps AND the loss on the train is:1117.3421630859375\n",
      "0.8163\n",
      "the 45000 setps AND the loss on the train is:1104.9722900390625\n",
      "0.8209\n",
      "the 45500 setps AND the loss on the train is:1093.0623779296875\n",
      "0.8213\n",
      "the 46000 setps AND the loss on the train is:1081.2596435546875\n",
      "0.8189\n",
      "the 46500 setps AND the loss on the train is:1069.321533203125\n",
      "0.8225\n",
      "the 47000 setps AND the loss on the train is:1057.8140869140625\n",
      "0.8228\n",
      "the 47500 setps AND the loss on the train is:1046.2935791015625\n",
      "0.8246\n",
      "the 48000 setps AND the loss on the train is:1034.8404541015625\n",
      "0.8268\n",
      "the 48500 setps AND the loss on the train is:1023.8284301757812\n",
      "0.825\n",
      "the 49000 setps AND the loss on the train is:1012.591796875\n",
      "0.8278\n",
      "the 49500 setps AND the loss on the train is:1001.662353515625\n",
      "0.8274\n",
      "the 50000 setps AND the loss on the train is:990.6312255859375\n",
      "0.8281\n",
      "the 50500 setps AND the loss on the train is:979.6890258789062\n",
      "0.8245\n",
      "the 51000 setps AND the loss on the train is:969.178466796875\n",
      "0.8285\n",
      "the 51500 setps AND the loss on the train is:958.31201171875\n",
      "0.8322\n",
      "the 52000 setps AND the loss on the train is:947.9295043945312\n",
      "0.8268\n",
      "the 52500 setps AND the loss on the train is:937.747314453125\n",
      "0.8308\n",
      "the 53000 setps AND the loss on the train is:927.5254516601562\n",
      "0.8326\n",
      "the 53500 setps AND the loss on the train is:917.457763671875\n",
      "0.8307\n",
      "the 54000 setps AND the loss on the train is:907.1575927734375\n",
      "0.8371\n",
      "the 54500 setps AND the loss on the train is:897.2997436523438\n",
      "0.835\n",
      "the 55000 setps AND the loss on the train is:887.5425415039062\n",
      "0.8364\n",
      "the 55500 setps AND the loss on the train is:877.9255981445312\n",
      "0.8355\n",
      "the 56000 setps AND the loss on the train is:868.0105590820312\n",
      "0.8389\n",
      "the 56500 setps AND the loss on the train is:858.7321166992188\n",
      "0.8395\n",
      "the 57000 setps AND the loss on the train is:849.1188354492188\n",
      "0.8381\n",
      "the 57500 setps AND the loss on the train is:839.8693237304688\n",
      "0.8337\n",
      "the 58000 setps AND the loss on the train is:830.6091918945312\n",
      "0.8386\n",
      "the 58500 setps AND the loss on the train is:821.6561889648438\n",
      "0.8408\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 59000 setps AND the loss on the train is:812.67822265625\n",
      "0.8409\n",
      "the 59500 setps AND the loss on the train is:803.7282104492188\n",
      "0.843\n",
      "the 60000 setps AND the loss on the train is:794.708740234375\n",
      "0.8427\n",
      "the 60500 setps AND the loss on the train is:786.1716918945312\n",
      "0.842\n",
      "the 61000 setps AND the loss on the train is:777.3434448242188\n",
      "0.8456\n",
      "the 61500 setps AND the loss on the train is:768.8237915039062\n",
      "0.8455\n",
      "the 62000 setps AND the loss on the train is:760.2908325195312\n",
      "0.8455\n",
      "the 62500 setps AND the loss on the train is:752.1152954101562\n",
      "0.8456\n",
      "the 63000 setps AND the loss on the train is:743.7379150390625\n",
      "0.8473\n",
      "the 63500 setps AND the loss on the train is:735.597412109375\n",
      "0.8494\n",
      "the 64000 setps AND the loss on the train is:727.266845703125\n",
      "0.8515\n",
      "the 64500 setps AND the loss on the train is:719.5782470703125\n",
      "0.8469\n",
      "the 65000 setps AND the loss on the train is:711.5438842773438\n",
      "0.8493\n",
      "the 65500 setps AND the loss on the train is:703.67822265625\n",
      "0.8497\n",
      "the 66000 setps AND the loss on the train is:696.1159057617188\n",
      "0.8497\n",
      "the 66500 setps AND the loss on the train is:688.4779052734375\n",
      "0.85\n",
      "the 67000 setps AND the loss on the train is:680.8173217773438\n",
      "0.8523\n",
      "the 67500 setps AND the loss on the train is:673.2227783203125\n",
      "0.8531\n",
      "the 68000 setps AND the loss on the train is:665.7794189453125\n",
      "0.852\n",
      "the 68500 setps AND the loss on the train is:658.74560546875\n",
      "0.8536\n",
      "the 69000 setps AND the loss on the train is:651.1983032226562\n",
      "0.8528\n",
      "the 69500 setps AND the loss on the train is:644.2022094726562\n",
      "0.8544\n",
      "the 70000 setps AND the loss on the train is:637.2342529296875\n",
      "0.8554\n",
      "the 70500 setps AND the loss on the train is:629.9930419921875\n",
      "0.8567\n",
      "the 71000 setps AND the loss on the train is:623.1856079101562\n",
      "0.8559\n",
      "the 71500 setps AND the loss on the train is:616.173583984375\n",
      "0.8577\n",
      "the 72000 setps AND the loss on the train is:609.5494995117188\n",
      "0.8586\n",
      "the 72500 setps AND the loss on the train is:602.9098510742188\n",
      "0.86\n",
      "the 73000 setps AND the loss on the train is:596.2037963867188\n",
      "0.8616\n",
      "the 73500 setps AND the loss on the train is:589.8312377929688\n",
      "0.8606\n",
      "the 74000 setps AND the loss on the train is:583.2488403320312\n",
      "0.8612\n",
      "the 74500 setps AND the loss on the train is:576.9471435546875\n",
      "0.8623\n",
      "the 75000 setps AND the loss on the train is:570.5286865234375\n",
      "0.8634\n",
      "the 75500 setps AND the loss on the train is:564.2542724609375\n",
      "0.8626\n",
      "the 76000 setps AND the loss on the train is:557.9976196289062\n",
      "0.8614\n",
      "the 76500 setps AND the loss on the train is:551.9490966796875\n",
      "0.8643\n",
      "the 77000 setps AND the loss on the train is:545.9506225585938\n",
      "0.8631\n",
      "the 77500 setps AND the loss on the train is:539.8685302734375\n",
      "0.8647\n",
      "the 78000 setps AND the loss on the train is:534.1180419921875\n",
      "0.8659\n",
      "the 78500 setps AND the loss on the train is:528.0997314453125\n",
      "0.8668\n",
      "the 79000 setps AND the loss on the train is:522.4202270507812\n",
      "0.8674\n",
      "the 79500 setps AND the loss on the train is:516.6874389648438\n",
      "0.8666\n",
      "the 80000 setps AND the loss on the train is:511.0718688964844\n",
      "0.8695\n",
      "the 80500 setps AND the loss on the train is:505.5904235839844\n",
      "0.869\n",
      "the 81000 setps AND the loss on the train is:500.08349609375\n",
      "0.869\n",
      "the 81500 setps AND the loss on the train is:494.50177001953125\n",
      "0.8698\n",
      "the 82000 setps AND the loss on the train is:489.0953063964844\n",
      "0.8698\n",
      "the 82500 setps AND the loss on the train is:483.82611083984375\n",
      "0.8706\n",
      "the 83000 setps AND the loss on the train is:478.5875244140625\n",
      "0.8726\n",
      "the 83500 setps AND the loss on the train is:473.2974853515625\n",
      "0.8712\n",
      "the 84000 setps AND the loss on the train is:468.2748718261719\n",
      "0.8725\n",
      "the 84500 setps AND the loss on the train is:463.01983642578125\n",
      "0.8723\n",
      "the 85000 setps AND the loss on the train is:458.1025085449219\n",
      "0.873\n",
      "the 85500 setps AND the loss on the train is:453.1274108886719\n",
      "0.873\n",
      "the 86000 setps AND the loss on the train is:448.0427551269531\n",
      "0.8736\n",
      "the 86500 setps AND the loss on the train is:443.37286376953125\n",
      "0.8742\n",
      "the 87000 setps AND the loss on the train is:438.5402526855469\n",
      "0.8763\n",
      "the 87500 setps AND the loss on the train is:433.9337463378906\n",
      "0.875\n",
      "the 88000 setps AND the loss on the train is:429.0607604980469\n",
      "0.8753\n",
      "the 88500 setps AND the loss on the train is:424.3050231933594\n",
      "0.8758\n",
      "the 89000 setps AND the loss on the train is:419.75421142578125\n",
      "0.876\n",
      "the 89500 setps AND the loss on the train is:415.2829895019531\n",
      "0.8766\n",
      "the 90000 setps AND the loss on the train is:410.6339111328125\n",
      "0.8775\n",
      "the 90500 setps AND the loss on the train is:406.33197021484375\n",
      "0.8768\n",
      "the 91000 setps AND the loss on the train is:401.950439453125\n",
      "0.8776\n",
      "the 91500 setps AND the loss on the train is:397.76806640625\n",
      "0.8788\n",
      "the 92000 setps AND the loss on the train is:393.328369140625\n",
      "0.8801\n",
      "the 92500 setps AND the loss on the train is:389.1934814453125\n",
      "0.8798\n",
      "the 93000 setps AND the loss on the train is:385.0864562988281\n",
      "0.8805\n",
      "the 93500 setps AND the loss on the train is:381.0423889160156\n",
      "0.8811\n",
      "the 94000 setps AND the loss on the train is:376.8269348144531\n",
      "0.8805\n",
      "the 94500 setps AND the loss on the train is:372.7751770019531\n",
      "0.882\n",
      "the 95000 setps AND the loss on the train is:368.6247863769531\n",
      "0.8817\n",
      "the 95500 setps AND the loss on the train is:364.8139953613281\n",
      "0.8822\n",
      "the 96000 setps AND the loss on the train is:360.9347229003906\n",
      "0.8834\n",
      "the 96500 setps AND the loss on the train is:356.9993591308594\n",
      "0.8833\n",
      "the 97000 setps AND the loss on the train is:353.2425231933594\n",
      "0.883\n",
      "the 97500 setps AND the loss on the train is:349.4827880859375\n",
      "0.8837\n",
      "the 98000 setps AND the loss on the train is:345.8631286621094\n",
      "0.8842\n",
      "the 98500 setps AND the loss on the train is:342.02728271484375\n",
      "0.8844\n",
      "the 99000 setps AND the loss on the train is:338.40972900390625\n",
      "0.8841\n",
      "the 99500 setps AND the loss on the train is:334.8652648925781\n",
      "0.8854\n",
      "the 100000 setps AND the loss on the train is:331.30975341796875\n",
      "0.8844\n",
      "the 100500 setps AND the loss on the train is:327.7810974121094\n",
      "0.8855\n",
      "the 101000 setps AND the loss on the train is:324.27978515625\n",
      "0.8867\n",
      "the 101500 setps AND the loss on the train is:320.83856201171875\n",
      "0.8868\n",
      "the 102000 setps AND the loss on the train is:317.3620910644531\n",
      "0.8877\n",
      "the 102500 setps AND the loss on the train is:314.091796875\n",
      "0.8874\n",
      "the 103000 setps AND the loss on the train is:310.78546142578125\n",
      "0.8886\n",
      "the 103500 setps AND the loss on the train is:307.4726867675781\n",
      "0.8881\n",
      "the 104000 setps AND the loss on the train is:304.4210510253906\n",
      "0.8899\n",
      "the 104500 setps AND the loss on the train is:300.9084777832031\n",
      "0.8891\n",
      "the 105000 setps AND the loss on the train is:297.8338623046875\n",
      "0.8897\n",
      "the 105500 setps AND the loss on the train is:294.7525329589844\n",
      "0.8897\n",
      "the 106000 setps AND the loss on the train is:291.7123107910156\n",
      "0.89\n",
      "the 106500 setps AND the loss on the train is:288.53631591796875\n",
      "0.8922\n",
      "the 107000 setps AND the loss on the train is:285.537109375\n",
      "0.8911\n",
      "the 107500 setps AND the loss on the train is:282.63787841796875\n",
      "0.8911\n",
      "the 108000 setps AND the loss on the train is:279.5399169921875\n",
      "0.8911\n",
      "the 108500 setps AND the loss on the train is:276.6805114746094\n",
      "0.8928\n",
      "the 109000 setps AND the loss on the train is:273.6669921875\n",
      "0.8935\n",
      "the 109500 setps AND the loss on the train is:270.9812927246094\n",
      "0.8931\n",
      "the 110000 setps AND the loss on the train is:268.119873046875\n",
      "0.8936\n",
      "the 110500 setps AND the loss on the train is:265.24627685546875\n",
      "0.8936\n",
      "the 111000 setps AND the loss on the train is:262.4121398925781\n",
      "0.894\n",
      "the 111500 setps AND the loss on the train is:259.65313720703125\n",
      "0.8939\n",
      "the 112000 setps AND the loss on the train is:256.969970703125\n",
      "0.8953\n",
      "the 112500 setps AND the loss on the train is:254.39157104492188\n",
      "0.895\n",
      "the 113000 setps AND the loss on the train is:251.63809204101562\n",
      "0.8954\n",
      "the 113500 setps AND the loss on the train is:249.0620574951172\n",
      "0.8938\n",
      "the 114000 setps AND the loss on the train is:246.36883544921875\n",
      "0.895\n",
      "the 114500 setps AND the loss on the train is:243.77862548828125\n",
      "0.8959\n",
      "the 115000 setps AND the loss on the train is:241.3327178955078\n",
      "0.8965\n",
      "the 115500 setps AND the loss on the train is:239.0498504638672\n",
      "0.8965\n",
      "the 116000 setps AND the loss on the train is:236.42674255371094\n",
      "0.897\n",
      "the 116500 setps AND the loss on the train is:233.8263397216797\n",
      "0.8974\n",
      "the 117000 setps AND the loss on the train is:231.34005737304688\n",
      "0.8971\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 117500 setps AND the loss on the train is:228.9676055908203\n",
      "0.8977\n",
      "the 118000 setps AND the loss on the train is:226.48228454589844\n",
      "0.8976\n",
      "the 118500 setps AND the loss on the train is:224.19180297851562\n",
      "0.8993\n",
      "the 119000 setps AND the loss on the train is:221.90658569335938\n",
      "0.8985\n",
      "the 119500 setps AND the loss on the train is:219.61184692382812\n",
      "0.8991\n",
      "the 120000 setps AND the loss on the train is:217.308837890625\n",
      "0.8999\n",
      "the 120500 setps AND the loss on the train is:215.00637817382812\n",
      "0.8979\n",
      "the 121000 setps AND the loss on the train is:212.7970733642578\n",
      "0.9012\n",
      "the 121500 setps AND the loss on the train is:210.62887573242188\n",
      "0.8996\n",
      "the 122000 setps AND the loss on the train is:208.56069946289062\n",
      "0.9015\n",
      "the 122500 setps AND the loss on the train is:206.19216918945312\n",
      "0.9006\n",
      "the 123000 setps AND the loss on the train is:204.04881286621094\n",
      "0.9008\n",
      "the 123500 setps AND the loss on the train is:201.92218017578125\n",
      "0.9\n",
      "the 124000 setps AND the loss on the train is:200.09967041015625\n",
      "0.9017\n",
      "the 124500 setps AND the loss on the train is:197.8101043701172\n",
      "0.9024\n",
      "the 125000 setps AND the loss on the train is:195.78048706054688\n",
      "0.9017\n",
      "the 125500 setps AND the loss on the train is:193.56118774414062\n",
      "0.9018\n",
      "the 126000 setps AND the loss on the train is:191.59381103515625\n",
      "0.903\n",
      "the 126500 setps AND the loss on the train is:189.63279724121094\n",
      "0.9028\n",
      "the 127000 setps AND the loss on the train is:187.8448486328125\n",
      "0.9022\n",
      "the 127500 setps AND the loss on the train is:185.70326232910156\n",
      "0.9033\n",
      "the 128000 setps AND the loss on the train is:183.8070068359375\n",
      "0.9049\n",
      "the 128500 setps AND the loss on the train is:181.90989685058594\n",
      "0.9049\n",
      "the 129000 setps AND the loss on the train is:179.99172973632812\n",
      "0.9047\n",
      "the 129500 setps AND the loss on the train is:178.26683044433594\n",
      "0.9051\n",
      "the 130000 setps AND the loss on the train is:176.41041564941406\n",
      "0.9056\n",
      "the 130500 setps AND the loss on the train is:174.39305114746094\n",
      "0.9068\n",
      "the 131000 setps AND the loss on the train is:172.71583557128906\n",
      "0.9064\n",
      "the 131500 setps AND the loss on the train is:170.82528686523438\n",
      "0.9068\n",
      "the 132000 setps AND the loss on the train is:169.20364379882812\n",
      "0.907\n",
      "the 132500 setps AND the loss on the train is:167.29087829589844\n",
      "0.9074\n",
      "the 133000 setps AND the loss on the train is:165.56605529785156\n",
      "0.9066\n",
      "the 133500 setps AND the loss on the train is:163.8369140625\n",
      "0.9076\n",
      "the 134000 setps AND the loss on the train is:162.17466735839844\n",
      "0.9085\n",
      "the 134500 setps AND the loss on the train is:160.41871643066406\n",
      "0.9089\n",
      "the 135000 setps AND the loss on the train is:158.62789916992188\n",
      "0.9092\n",
      "the 135500 setps AND the loss on the train is:157.0037078857422\n",
      "0.9093\n",
      "the 136000 setps AND the loss on the train is:155.51644897460938\n",
      "0.9093\n",
      "the 136500 setps AND the loss on the train is:153.7003631591797\n",
      "0.9099\n",
      "the 137000 setps AND the loss on the train is:152.0748291015625\n",
      "0.9096\n",
      "the 137500 setps AND the loss on the train is:150.4802703857422\n",
      "0.9102\n",
      "the 138000 setps AND the loss on the train is:148.8785400390625\n",
      "0.9102\n",
      "the 138500 setps AND the loss on the train is:147.3284912109375\n",
      "0.9104\n",
      "the 139000 setps AND the loss on the train is:145.85520935058594\n",
      "0.911\n",
      "the 139500 setps AND the loss on the train is:144.3864288330078\n",
      "0.9109\n",
      "the 140000 setps AND the loss on the train is:142.74087524414062\n",
      "0.9104\n",
      "the 140500 setps AND the loss on the train is:141.25692749023438\n",
      "0.912\n",
      "the 141000 setps AND the loss on the train is:139.95217895507812\n",
      "0.9113\n",
      "the 141500 setps AND the loss on the train is:138.3282470703125\n",
      "0.9114\n",
      "the 142000 setps AND the loss on the train is:136.82997131347656\n",
      "0.9115\n",
      "the 142500 setps AND the loss on the train is:135.30355834960938\n",
      "0.9116\n",
      "the 143000 setps AND the loss on the train is:133.96080017089844\n",
      "0.9118\n",
      "the 143500 setps AND the loss on the train is:132.5357208251953\n",
      "0.9119\n",
      "the 144000 setps AND the loss on the train is:130.99575805664062\n",
      "0.9124\n",
      "the 144500 setps AND the loss on the train is:129.8188934326172\n",
      "0.9124\n",
      "the 145000 setps AND the loss on the train is:128.30059814453125\n",
      "0.9133\n",
      "the 145500 setps AND the loss on the train is:127.01141357421875\n",
      "0.9132\n",
      "the 146000 setps AND the loss on the train is:125.60566711425781\n",
      "0.9138\n",
      "the 146500 setps AND the loss on the train is:124.3326187133789\n",
      "0.9132\n",
      "the 147000 setps AND the loss on the train is:122.94979858398438\n",
      "0.9146\n",
      "the 147500 setps AND the loss on the train is:121.4559555053711\n",
      "0.9154\n",
      "the 148000 setps AND the loss on the train is:120.28984832763672\n",
      "0.9149\n",
      "the 148500 setps AND the loss on the train is:118.96662902832031\n",
      "0.9141\n",
      "the 149000 setps AND the loss on the train is:117.78660583496094\n",
      "0.9155\n",
      "the 149500 setps AND the loss on the train is:116.43630981445312\n",
      "0.9142\n",
      "the 150000 setps AND the loss on the train is:115.2881851196289\n",
      "0.9148\n",
      "the 150500 setps AND the loss on the train is:114.06144714355469\n",
      "0.9158\n",
      "the 151000 setps AND the loss on the train is:112.68037414550781\n",
      "0.9152\n",
      "the 151500 setps AND the loss on the train is:111.4946517944336\n",
      "0.9161\n",
      "the 152000 setps AND the loss on the train is:110.41036224365234\n",
      "0.9161\n",
      "the 152500 setps AND the loss on the train is:109.21427917480469\n",
      "0.9166\n",
      "the 153000 setps AND the loss on the train is:108.05020904541016\n",
      "0.9163\n",
      "the 153500 setps AND the loss on the train is:106.7635726928711\n",
      "0.9167\n",
      "the 154000 setps AND the loss on the train is:105.69754791259766\n",
      "0.9169\n",
      "the 154500 setps AND the loss on the train is:104.4626235961914\n",
      "0.9159\n",
      "the 155000 setps AND the loss on the train is:103.34654235839844\n",
      "0.918\n",
      "the 155500 setps AND the loss on the train is:102.0877685546875\n",
      "0.9172\n",
      "the 156000 setps AND the loss on the train is:101.0584945678711\n",
      "0.918\n",
      "the 156500 setps AND the loss on the train is:99.9714126586914\n",
      "0.9174\n",
      "the 157000 setps AND the loss on the train is:98.85614776611328\n",
      "0.9175\n",
      "the 157500 setps AND the loss on the train is:97.81411743164062\n",
      "0.9182\n",
      "the 158000 setps AND the loss on the train is:96.70602416992188\n",
      "0.9178\n",
      "the 158500 setps AND the loss on the train is:95.81117248535156\n",
      "0.9178\n",
      "the 159000 setps AND the loss on the train is:94.72661590576172\n",
      "0.9188\n",
      "the 159500 setps AND the loss on the train is:93.61945343017578\n",
      "0.9188\n",
      "the 160000 setps AND the loss on the train is:92.64935302734375\n",
      "0.918\n",
      "the 160500 setps AND the loss on the train is:91.57523345947266\n",
      "0.9186\n",
      "the 161000 setps AND the loss on the train is:90.52234649658203\n",
      "0.9189\n",
      "the 161500 setps AND the loss on the train is:89.60210418701172\n",
      "0.9185\n",
      "the 162000 setps AND the loss on the train is:88.62699890136719\n",
      "0.9187\n",
      "the 162500 setps AND the loss on the train is:87.56588745117188\n",
      "0.9196\n",
      "the 163000 setps AND the loss on the train is:86.71556091308594\n",
      "0.9193\n",
      "the 163500 setps AND the loss on the train is:85.72932434082031\n",
      "0.9193\n",
      "the 164000 setps AND the loss on the train is:84.7714614868164\n",
      "0.9196\n",
      "the 164500 setps AND the loss on the train is:83.8294906616211\n",
      "0.9194\n",
      "the 165000 setps AND the loss on the train is:82.85795593261719\n",
      "0.9194\n",
      "the 165500 setps AND the loss on the train is:82.15062713623047\n",
      "0.9198\n",
      "the 166000 setps AND the loss on the train is:81.17953491210938\n",
      "0.9208\n",
      "the 166500 setps AND the loss on the train is:80.23121643066406\n",
      "0.9206\n",
      "the 167000 setps AND the loss on the train is:79.36946105957031\n",
      "0.9205\n",
      "the 167500 setps AND the loss on the train is:78.44579315185547\n",
      "0.9198\n",
      "the 168000 setps AND the loss on the train is:77.56974792480469\n",
      "0.9211\n",
      "the 168500 setps AND the loss on the train is:76.83833312988281\n",
      "0.9207\n",
      "the 169000 setps AND the loss on the train is:75.8361587524414\n",
      "0.9214\n",
      "the 169500 setps AND the loss on the train is:75.06243133544922\n",
      "0.9202\n",
      "the 170000 setps AND the loss on the train is:74.12349700927734\n",
      "0.9208\n",
      "the 170500 setps AND the loss on the train is:73.45391082763672\n",
      "0.9204\n",
      "the 171000 setps AND the loss on the train is:72.57635498046875\n",
      "0.9213\n",
      "the 171500 setps AND the loss on the train is:71.68592071533203\n",
      "0.9208\n",
      "the 172000 setps AND the loss on the train is:70.86798095703125\n",
      "0.9217\n",
      "the 172500 setps AND the loss on the train is:70.06818389892578\n",
      "0.9214\n",
      "the 173000 setps AND the loss on the train is:69.34373474121094\n",
      "0.9217\n",
      "the 173500 setps AND the loss on the train is:68.61829376220703\n",
      "0.9221\n",
      "the 174000 setps AND the loss on the train is:67.8973159790039\n",
      "0.9222\n",
      "the 174500 setps AND the loss on the train is:67.12774658203125\n",
      "0.9222\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 175000 setps AND the loss on the train is:66.32713317871094\n",
      "0.9215\n",
      "the 175500 setps AND the loss on the train is:65.55683898925781\n",
      "0.9217\n",
      "the 176000 setps AND the loss on the train is:64.78273010253906\n",
      "0.9218\n",
      "the 176500 setps AND the loss on the train is:64.16673278808594\n",
      "0.9212\n",
      "the 177000 setps AND the loss on the train is:63.266395568847656\n",
      "0.9228\n",
      "the 177500 setps AND the loss on the train is:62.62837600708008\n",
      "0.922\n",
      "the 178000 setps AND the loss on the train is:61.865596771240234\n",
      "0.9217\n",
      "the 178500 setps AND the loss on the train is:61.22690200805664\n",
      "0.922\n",
      "the 179000 setps AND the loss on the train is:60.7728157043457\n",
      "0.923\n",
      "the 179500 setps AND the loss on the train is:59.84741973876953\n",
      "0.922\n",
      "the 180000 setps AND the loss on the train is:59.1158332824707\n",
      "0.923\n",
      "the 180500 setps AND the loss on the train is:58.56059646606445\n",
      "0.9229\n",
      "the 181000 setps AND the loss on the train is:57.838443756103516\n",
      "0.9228\n",
      "the 181500 setps AND the loss on the train is:57.11701965332031\n",
      "0.9238\n",
      "the 182000 setps AND the loss on the train is:56.54899978637695\n",
      "0.9236\n",
      "the 182500 setps AND the loss on the train is:55.9403190612793\n",
      "0.9241\n",
      "the 183000 setps AND the loss on the train is:55.18992233276367\n",
      "0.9243\n",
      "the 183500 setps AND the loss on the train is:54.70700454711914\n",
      "0.9242\n",
      "the 184000 setps AND the loss on the train is:53.978553771972656\n",
      "0.9239\n",
      "the 184500 setps AND the loss on the train is:53.244903564453125\n",
      "0.9245\n",
      "the 185000 setps AND the loss on the train is:52.839054107666016\n",
      "0.9239\n",
      "the 185500 setps AND the loss on the train is:52.08279800415039\n",
      "0.9241\n",
      "the 186000 setps AND the loss on the train is:51.522029876708984\n",
      "0.9252\n",
      "the 186500 setps AND the loss on the train is:51.00010681152344\n",
      "0.9245\n",
      "the 187000 setps AND the loss on the train is:50.475433349609375\n",
      "0.9253\n",
      "the 187500 setps AND the loss on the train is:49.9234504699707\n",
      "0.9258\n",
      "the 188000 setps AND the loss on the train is:49.36286163330078\n",
      "0.9256\n",
      "the 188500 setps AND the loss on the train is:48.54845428466797\n",
      "0.9251\n",
      "the 189000 setps AND the loss on the train is:48.079524993896484\n",
      "0.9254\n",
      "the 189500 setps AND the loss on the train is:47.59615707397461\n",
      "0.9257\n",
      "the 190000 setps AND the loss on the train is:47.01640319824219\n",
      "0.9258\n",
      "the 190500 setps AND the loss on the train is:46.384639739990234\n",
      "0.9255\n",
      "the 191000 setps AND the loss on the train is:45.87906265258789\n",
      "0.9252\n",
      "the 191500 setps AND the loss on the train is:45.319091796875\n",
      "0.9253\n",
      "the 192000 setps AND the loss on the train is:44.7405891418457\n",
      "0.9255\n",
      "the 192500 setps AND the loss on the train is:44.28125\n",
      "0.9262\n",
      "the 193000 setps AND the loss on the train is:43.76430130004883\n",
      "0.9259\n",
      "the 193500 setps AND the loss on the train is:43.327354431152344\n",
      "0.926\n",
      "the 194000 setps AND the loss on the train is:42.88211441040039\n",
      "0.9256\n",
      "the 194500 setps AND the loss on the train is:42.2813835144043\n",
      "0.9254\n",
      "the 195000 setps AND the loss on the train is:41.72134017944336\n",
      "0.9263\n",
      "the 195500 setps AND the loss on the train is:41.29549026489258\n",
      "0.9266\n",
      "the 196000 setps AND the loss on the train is:40.96991729736328\n",
      "0.927\n",
      "the 196500 setps AND the loss on the train is:40.353240966796875\n",
      "0.9263\n",
      "the 197000 setps AND the loss on the train is:39.965545654296875\n",
      "0.9265\n",
      "the 197500 setps AND the loss on the train is:39.374839782714844\n",
      "0.9266\n",
      "the 198000 setps AND the loss on the train is:38.91503143310547\n",
      "0.9267\n",
      "the 198500 setps AND the loss on the train is:38.44380569458008\n",
      "0.9265\n",
      "the 199000 setps AND the loss on the train is:38.0061149597168\n",
      "0.9272\n",
      "the 199500 setps AND the loss on the train is:37.60493850708008\n",
      "0.9264\n",
      "the 200000 setps AND the loss on the train is:37.225318908691406\n",
      "0.9268\n",
      "the 200500 setps AND the loss on the train is:36.72091293334961\n",
      "0.9267\n",
      "the 201000 setps AND the loss on the train is:36.361270904541016\n",
      "0.9268\n",
      "the 201500 setps AND the loss on the train is:35.87194061279297\n",
      "0.9275\n",
      "the 202000 setps AND the loss on the train is:35.58082580566406\n",
      "0.9277\n",
      "the 202500 setps AND the loss on the train is:35.31342697143555\n",
      "0.9275\n",
      "the 203000 setps AND the loss on the train is:34.604488372802734\n",
      "0.9271\n",
      "the 203500 setps AND the loss on the train is:34.17174530029297\n",
      "0.9279\n",
      "the 204000 setps AND the loss on the train is:33.88311004638672\n",
      "0.9277\n",
      "the 204500 setps AND the loss on the train is:33.46012878417969\n",
      "0.9275\n",
      "the 205000 setps AND the loss on the train is:33.03990936279297\n",
      "0.9271\n",
      "the 205500 setps AND the loss on the train is:32.60208511352539\n",
      "0.9278\n",
      "the 206000 setps AND the loss on the train is:32.33415603637695\n",
      "0.9275\n",
      "the 206500 setps AND the loss on the train is:31.944442749023438\n",
      "0.928\n",
      "the 207000 setps AND the loss on the train is:31.562602996826172\n",
      "0.928\n",
      "the 207500 setps AND the loss on the train is:31.17697525024414\n",
      "0.9276\n",
      "the 208000 setps AND the loss on the train is:30.80722427368164\n",
      "0.9287\n",
      "the 208500 setps AND the loss on the train is:30.461650848388672\n",
      "0.9276\n",
      "the 209000 setps AND the loss on the train is:30.06449317932129\n",
      "0.9282\n",
      "the 209500 setps AND the loss on the train is:29.70282745361328\n",
      "0.9279\n",
      "the 210000 setps AND the loss on the train is:29.41407012939453\n",
      "0.9274\n",
      "the 210500 setps AND the loss on the train is:29.23491668701172\n",
      "0.9277\n",
      "the 211000 setps AND the loss on the train is:28.701515197753906\n",
      "0.9288\n",
      "the 211500 setps AND the loss on the train is:28.345382690429688\n",
      "0.9283\n",
      "the 212000 setps AND the loss on the train is:27.954200744628906\n",
      "0.9277\n",
      "the 212500 setps AND the loss on the train is:27.829465866088867\n",
      "0.9284\n",
      "the 213000 setps AND the loss on the train is:27.356321334838867\n",
      "0.9279\n",
      "the 213500 setps AND the loss on the train is:27.003990173339844\n",
      "0.9281\n",
      "the 214000 setps AND the loss on the train is:26.71485137939453\n",
      "0.9281\n",
      "the 214500 setps AND the loss on the train is:26.424665451049805\n",
      "0.9289\n",
      "the 215000 setps AND the loss on the train is:26.134273529052734\n",
      "0.928\n",
      "the 215500 setps AND the loss on the train is:25.843595504760742\n",
      "0.928\n",
      "the 216000 setps AND the loss on the train is:25.507061004638672\n",
      "0.9289\n",
      "the 216500 setps AND the loss on the train is:25.31841468811035\n",
      "0.9287\n",
      "the 217000 setps AND the loss on the train is:24.918170928955078\n",
      "0.9283\n",
      "the 217500 setps AND the loss on the train is:24.47528076171875\n",
      "0.9285\n",
      "the 218000 setps AND the loss on the train is:24.284610748291016\n",
      "0.9285\n",
      "the 218500 setps AND the loss on the train is:23.970516204833984\n",
      "0.9293\n",
      "the 219000 setps AND the loss on the train is:23.725984573364258\n",
      "0.9286\n",
      "the 219500 setps AND the loss on the train is:23.516000747680664\n",
      "0.9291\n",
      "the 220000 setps AND the loss on the train is:23.06139373779297\n",
      "0.929\n",
      "the 220500 setps AND the loss on the train is:22.80184555053711\n",
      "0.9289\n",
      "the 221000 setps AND the loss on the train is:22.589862823486328\n",
      "0.9289\n",
      "the 221500 setps AND the loss on the train is:22.32313346862793\n",
      "0.9303\n",
      "the 222000 setps AND the loss on the train is:22.038179397583008\n",
      "0.93\n",
      "the 222500 setps AND the loss on the train is:21.707529067993164\n",
      "0.9294\n",
      "the 223000 setps AND the loss on the train is:21.66105842590332\n",
      "0.9306\n",
      "the 223500 setps AND the loss on the train is:21.1468563079834\n",
      "0.9296\n",
      "the 224000 setps AND the loss on the train is:21.022613525390625\n",
      "0.9296\n",
      "the 224500 setps AND the loss on the train is:20.691925048828125\n",
      "0.9302\n",
      "the 225000 setps AND the loss on the train is:20.48982048034668\n",
      "0.93\n",
      "the 225500 setps AND the loss on the train is:20.30536651611328\n",
      "0.9305\n",
      "the 226000 setps AND the loss on the train is:20.149490356445312\n",
      "0.931\n",
      "the 226500 setps AND the loss on the train is:19.84444236755371\n",
      "0.9305\n",
      "the 227000 setps AND the loss on the train is:19.553863525390625\n",
      "0.9314\n",
      "the 227500 setps AND the loss on the train is:19.247575759887695\n",
      "0.9314\n",
      "the 228000 setps AND the loss on the train is:19.039525985717773\n",
      "0.9305\n",
      "the 228500 setps AND the loss on the train is:18.74040985107422\n",
      "0.9306\n",
      "the 229000 setps AND the loss on the train is:18.606990814208984\n",
      "0.9313\n",
      "the 229500 setps AND the loss on the train is:18.480024337768555\n",
      "0.931\n",
      "the 230000 setps AND the loss on the train is:18.1778564453125\n",
      "0.9315\n",
      "the 230500 setps AND the loss on the train is:17.890703201293945\n",
      "0.9312\n",
      "the 231000 setps AND the loss on the train is:17.749603271484375\n",
      "0.9315\n",
      "the 231500 setps AND the loss on the train is:17.672008514404297\n",
      "0.932\n",
      "the 232000 setps AND the loss on the train is:17.246292114257812\n",
      "0.9319\n",
      "the 232500 setps AND the loss on the train is:17.224353790283203\n",
      "0.932\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 233000 setps AND the loss on the train is:16.900218963623047\n",
      "0.9315\n",
      "the 233500 setps AND the loss on the train is:16.746131896972656\n",
      "0.9317\n",
      "the 234000 setps AND the loss on the train is:16.51491355895996\n",
      "0.9316\n",
      "the 234500 setps AND the loss on the train is:16.445627212524414\n",
      "0.9313\n",
      "the 235000 setps AND the loss on the train is:16.160743713378906\n",
      "0.9318\n",
      "the 235500 setps AND the loss on the train is:15.867022514343262\n",
      "0.9316\n",
      "the 236000 setps AND the loss on the train is:15.637263298034668\n",
      "0.9319\n",
      "the 236500 setps AND the loss on the train is:15.533740997314453\n",
      "0.9321\n",
      "the 237000 setps AND the loss on the train is:15.393250465393066\n",
      "0.9325\n",
      "the 237500 setps AND the loss on the train is:15.196521759033203\n",
      "0.9317\n",
      "the 238000 setps AND the loss on the train is:14.942296981811523\n",
      "0.9327\n",
      "the 238500 setps AND the loss on the train is:14.824377059936523\n",
      "0.9315\n",
      "the 239000 setps AND the loss on the train is:14.663101196289062\n",
      "0.9324\n",
      "the 239500 setps AND the loss on the train is:14.359245300292969\n",
      "0.9324\n",
      "the 240000 setps AND the loss on the train is:14.272693634033203\n",
      "0.9328\n",
      "the 240500 setps AND the loss on the train is:14.151493072509766\n",
      "0.9326\n",
      "the 241000 setps AND the loss on the train is:14.001708030700684\n",
      "0.9325\n",
      "the 241500 setps AND the loss on the train is:13.791762351989746\n",
      "0.9326\n",
      "the 242000 setps AND the loss on the train is:13.59317398071289\n",
      "0.9325\n",
      "the 242500 setps AND the loss on the train is:13.453116416931152\n",
      "0.9324\n",
      "the 243000 setps AND the loss on the train is:13.42648983001709\n",
      "0.933\n",
      "the 243500 setps AND the loss on the train is:13.16385555267334\n",
      "0.9327\n",
      "the 244000 setps AND the loss on the train is:12.97936725616455\n",
      "0.9327\n",
      "the 244500 setps AND the loss on the train is:12.920241355895996\n",
      "0.9329\n",
      "the 245000 setps AND the loss on the train is:12.66236686706543\n",
      "0.9321\n",
      "the 245500 setps AND the loss on the train is:12.488821983337402\n",
      "0.9332\n",
      "the 246000 setps AND the loss on the train is:12.3819580078125\n",
      "0.9325\n",
      "the 246500 setps AND the loss on the train is:12.244879722595215\n",
      "0.9328\n",
      "the 247000 setps AND the loss on the train is:11.999505043029785\n",
      "0.9332\n",
      "the 247500 setps AND the loss on the train is:11.88582706451416\n",
      "0.9336\n",
      "the 248000 setps AND the loss on the train is:11.795771598815918\n",
      "0.9332\n",
      "the 248500 setps AND the loss on the train is:11.66521167755127\n",
      "0.9328\n",
      "the 249000 setps AND the loss on the train is:11.602656364440918\n",
      "0.9335\n",
      "the 249500 setps AND the loss on the train is:11.363113403320312\n",
      "0.9332\n",
      "the 250000 setps AND the loss on the train is:11.177393913269043\n",
      "0.9337\n",
      "the 250500 setps AND the loss on the train is:11.109465599060059\n",
      "0.9336\n",
      "the 251000 setps AND the loss on the train is:10.916940689086914\n",
      "0.934\n",
      "the 251500 setps AND the loss on the train is:10.861553192138672\n",
      "0.9335\n",
      "the 252000 setps AND the loss on the train is:10.714160919189453\n",
      "0.934\n",
      "the 252500 setps AND the loss on the train is:10.591341018676758\n",
      "0.934\n",
      "the 253000 setps AND the loss on the train is:10.52402114868164\n",
      "0.9337\n",
      "the 253500 setps AND the loss on the train is:10.324121475219727\n",
      "0.9336\n",
      "the 254000 setps AND the loss on the train is:10.420738220214844\n",
      "0.9334\n",
      "the 254500 setps AND the loss on the train is:10.034584045410156\n",
      "0.9339\n",
      "the 255000 setps AND the loss on the train is:10.037919044494629\n",
      "0.9339\n",
      "the 255500 setps AND the loss on the train is:9.78090763092041\n",
      "0.9344\n",
      "the 256000 setps AND the loss on the train is:9.72050666809082\n",
      "0.9338\n",
      "the 256500 setps AND the loss on the train is:9.568717956542969\n",
      "0.9336\n",
      "the 257000 setps AND the loss on the train is:9.645973205566406\n",
      "0.934\n",
      "the 257500 setps AND the loss on the train is:9.407611846923828\n",
      "0.9342\n",
      "the 258000 setps AND the loss on the train is:9.293777465820312\n",
      "0.9338\n",
      "the 258500 setps AND the loss on the train is:9.272847175598145\n",
      "0.9345\n",
      "the 259000 setps AND the loss on the train is:9.033349990844727\n",
      "0.9344\n",
      "the 259500 setps AND the loss on the train is:9.073955535888672\n",
      "0.9344\n",
      "the 260000 setps AND the loss on the train is:8.848428726196289\n",
      "0.9343\n",
      "the 260500 setps AND the loss on the train is:8.723000526428223\n",
      "0.9348\n",
      "the 261000 setps AND the loss on the train is:8.630260467529297\n",
      "0.9341\n",
      "the 261500 setps AND the loss on the train is:8.518770217895508\n",
      "0.934\n",
      "the 262000 setps AND the loss on the train is:8.411334037780762\n",
      "0.9343\n",
      "the 262500 setps AND the loss on the train is:8.380699157714844\n",
      "0.9341\n",
      "the 263000 setps AND the loss on the train is:8.265960693359375\n",
      "0.9343\n",
      "the 263500 setps AND the loss on the train is:8.285794258117676\n",
      "0.9343\n",
      "the 264000 setps AND the loss on the train is:8.012201309204102\n",
      "0.9344\n",
      "the 264500 setps AND the loss on the train is:7.883470058441162\n",
      "0.9342\n",
      "the 265000 setps AND the loss on the train is:7.871870040893555\n",
      "0.9351\n",
      "the 265500 setps AND the loss on the train is:7.925560474395752\n",
      "0.9348\n",
      "the 266000 setps AND the loss on the train is:7.686282634735107\n",
      "0.9349\n",
      "the 266500 setps AND the loss on the train is:7.596319675445557\n",
      "0.9348\n",
      "the 267000 setps AND the loss on the train is:7.6207661628723145\n",
      "0.935\n",
      "the 267500 setps AND the loss on the train is:7.419209957122803\n",
      "0.9351\n",
      "the 268000 setps AND the loss on the train is:7.36890172958374\n",
      "0.9356\n",
      "the 268500 setps AND the loss on the train is:7.323994159698486\n",
      "0.9348\n",
      "the 269000 setps AND the loss on the train is:7.228418350219727\n",
      "0.9343\n",
      "the 269500 setps AND the loss on the train is:7.080686092376709\n",
      "0.9355\n",
      "the 270000 setps AND the loss on the train is:7.033294677734375\n",
      "0.9356\n",
      "the 270500 setps AND the loss on the train is:7.020651817321777\n",
      "0.9351\n",
      "the 271000 setps AND the loss on the train is:6.8727850914001465\n",
      "0.9358\n",
      "the 271500 setps AND the loss on the train is:6.82868766784668\n",
      "0.9352\n",
      "the 272000 setps AND the loss on the train is:6.7224931716918945\n",
      "0.9355\n",
      "the 272500 setps AND the loss on the train is:6.594290256500244\n",
      "0.935\n",
      "the 273000 setps AND the loss on the train is:6.598105430603027\n",
      "0.9356\n",
      "the 273500 setps AND the loss on the train is:6.459505081176758\n",
      "0.936\n",
      "the 274000 setps AND the loss on the train is:6.478076934814453\n",
      "0.9351\n",
      "the 274500 setps AND the loss on the train is:6.41351318359375\n",
      "0.9357\n",
      "the 275000 setps AND the loss on the train is:6.21737813949585\n",
      "0.9356\n",
      "the 275500 setps AND the loss on the train is:6.227083206176758\n",
      "0.9355\n",
      "the 276000 setps AND the loss on the train is:6.0798516273498535\n",
      "0.9356\n",
      "the 276500 setps AND the loss on the train is:5.972181797027588\n",
      "0.9355\n",
      "the 277000 setps AND the loss on the train is:6.004871368408203\n",
      "0.9362\n",
      "the 277500 setps AND the loss on the train is:6.025252819061279\n",
      "0.9358\n",
      "the 278000 setps AND the loss on the train is:5.78247594833374\n",
      "0.936\n",
      "the 278500 setps AND the loss on the train is:5.821401119232178\n",
      "0.9357\n",
      "the 279000 setps AND the loss on the train is:5.863429069519043\n",
      "0.9354\n",
      "the 279500 setps AND the loss on the train is:5.708933353424072\n",
      "0.9362\n",
      "the 280000 setps AND the loss on the train is:5.496925354003906\n",
      "0.936\n",
      "the 280500 setps AND the loss on the train is:5.633932590484619\n",
      "0.9357\n",
      "the 281000 setps AND the loss on the train is:5.447575569152832\n",
      "0.9366\n",
      "the 281500 setps AND the loss on the train is:5.33293342590332\n",
      "0.9354\n",
      "the 282000 setps AND the loss on the train is:5.308751106262207\n",
      "0.9362\n",
      "the 282500 setps AND the loss on the train is:5.3039422035217285\n",
      "0.9364\n",
      "the 283000 setps AND the loss on the train is:5.183385848999023\n",
      "0.936\n",
      "the 283500 setps AND the loss on the train is:5.228665351867676\n",
      "0.9365\n",
      "the 284000 setps AND the loss on the train is:5.059689044952393\n",
      "0.9363\n",
      "the 284500 setps AND the loss on the train is:4.991481781005859\n",
      "0.9363\n",
      "the 285000 setps AND the loss on the train is:4.9159345626831055\n",
      "0.936\n",
      "the 285500 setps AND the loss on the train is:4.924030303955078\n",
      "0.9365\n",
      "the 286000 setps AND the loss on the train is:4.878232955932617\n",
      "0.9364\n",
      "the 286500 setps AND the loss on the train is:4.801495552062988\n",
      "0.9368\n",
      "the 287000 setps AND the loss on the train is:4.782104969024658\n",
      "0.9362\n",
      "the 287500 setps AND the loss on the train is:4.69743537902832\n",
      "0.9361\n",
      "the 288000 setps AND the loss on the train is:4.625001907348633\n",
      "0.9372\n",
      "the 288500 setps AND the loss on the train is:4.709072589874268\n",
      "0.9365\n",
      "the 289000 setps AND the loss on the train is:4.477755069732666\n",
      "0.9362\n",
      "the 289500 setps AND the loss on the train is:4.462526798248291\n",
      "0.9365\n",
      "the 290000 setps AND the loss on the train is:4.4209303855896\n",
      "0.9364\n",
      "the 290500 setps AND the loss on the train is:4.454113960266113\n",
      "0.9364\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 291000 setps AND the loss on the train is:4.345376491546631\n",
      "0.9369\n",
      "the 291500 setps AND the loss on the train is:4.257689476013184\n",
      "0.9365\n",
      "the 292000 setps AND the loss on the train is:4.238516330718994\n",
      "0.9368\n",
      "the 292500 setps AND the loss on the train is:4.185278415679932\n",
      "0.9373\n",
      "the 293000 setps AND the loss on the train is:4.203429222106934\n",
      "0.9371\n",
      "the 293500 setps AND the loss on the train is:4.141587257385254\n",
      "0.9371\n",
      "the 294000 setps AND the loss on the train is:4.069682598114014\n",
      "0.9362\n",
      "the 294500 setps AND the loss on the train is:4.04120397567749\n",
      "0.9369\n",
      "the 295000 setps AND the loss on the train is:3.9826204776763916\n",
      "0.9375\n",
      "the 295500 setps AND the loss on the train is:3.9570133686065674\n",
      "0.9367\n",
      "the 296000 setps AND the loss on the train is:3.9695241451263428\n",
      "0.9367\n",
      "the 296500 setps AND the loss on the train is:4.065787315368652\n",
      "0.9371\n",
      "the 297000 setps AND the loss on the train is:3.8240814208984375\n",
      "0.9367\n",
      "the 297500 setps AND the loss on the train is:3.823855400085449\n",
      "0.9375\n",
      "the 298000 setps AND the loss on the train is:3.760801315307617\n",
      "0.9366\n",
      "the 298500 setps AND the loss on the train is:3.7072484493255615\n",
      "0.9364\n",
      "the 299000 setps AND the loss on the train is:3.672128677368164\n",
      "0.9374\n",
      "the 299500 setps AND the loss on the train is:3.610358715057373\n",
      "0.9377\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,50],0.01)\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=get_weight([50,50],0.01)\n",
    "b2=tf.Variable(tf.random_normal([50]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.relu(logits2)\n",
    "\n",
    "w3=get_weight([50,10],0.01)\n",
    "b3=tf.Variable(tf.random_normal([10]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "\n",
    "\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits3))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(0.001).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits3,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(300000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4.3 在3.4.1基础上将学习率设定为指数衰减"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:4111.142578125\n",
      "0.1077\n",
      "the 500 setps AND the loss on the train is:1219.6956787109375\n",
      "0.3777\n",
      "the 1000 setps AND the loss on the train is:414.6470642089844\n",
      "0.7326\n",
      "the 1500 setps AND the loss on the train is:145.99359130859375\n",
      "0.8578\n",
      "the 2000 setps AND the loss on the train is:48.93886184692383\n",
      "0.9042\n",
      "the 2500 setps AND the loss on the train is:16.103031158447266\n",
      "0.9119\n",
      "the 3000 setps AND the loss on the train is:5.928826808929443\n",
      "0.922\n",
      "the 3500 setps AND the loss on the train is:3.085322856903076\n",
      "0.9251\n",
      "the 4000 setps AND the loss on the train is:1.9821949005126953\n",
      "0.9251\n",
      "the 4500 setps AND the loss on the train is:1.6856147050857544\n",
      "0.9296\n",
      "the 5000 setps AND the loss on the train is:1.631584882736206\n",
      "0.9319\n",
      "the 5500 setps AND the loss on the train is:1.5064136981964111\n",
      "0.9321\n",
      "the 6000 setps AND the loss on the train is:1.4905705451965332\n",
      "0.935\n",
      "the 6500 setps AND the loss on the train is:1.555361032485962\n",
      "0.9328\n",
      "the 7000 setps AND the loss on the train is:1.4634186029434204\n",
      "0.9414\n",
      "the 7500 setps AND the loss on the train is:1.5165867805480957\n",
      "0.9394\n",
      "the 8000 setps AND the loss on the train is:1.5212424993515015\n",
      "0.9382\n",
      "the 8500 setps AND the loss on the train is:1.4016677141189575\n",
      "0.9431\n",
      "the 9000 setps AND the loss on the train is:1.5091420412063599\n",
      "0.9385\n",
      "the 9500 setps AND the loss on the train is:1.4807090759277344\n",
      "0.9415\n",
      "the 10000 setps AND the loss on the train is:1.5605067014694214\n",
      "0.9415\n",
      "the 10500 setps AND the loss on the train is:1.4697843790054321\n",
      "0.942\n",
      "the 11000 setps AND the loss on the train is:1.4935188293457031\n",
      "0.9428\n",
      "the 11500 setps AND the loss on the train is:1.5292998552322388\n",
      "0.9436\n",
      "the 12000 setps AND the loss on the train is:1.4876899719238281\n",
      "0.9413\n",
      "the 12500 setps AND the loss on the train is:1.5359842777252197\n",
      "0.9432\n",
      "the 13000 setps AND the loss on the train is:1.583547830581665\n",
      "0.9457\n",
      "the 13500 setps AND the loss on the train is:1.50642728805542\n",
      "0.9455\n",
      "the 14000 setps AND the loss on the train is:1.4839539527893066\n",
      "0.9432\n",
      "the 14500 setps AND the loss on the train is:1.450748324394226\n",
      "0.9464\n",
      "the 15000 setps AND the loss on the train is:1.4114913940429688\n",
      "0.9447\n",
      "the 15500 setps AND the loss on the train is:1.5091476440429688\n",
      "0.9402\n",
      "the 16000 setps AND the loss on the train is:1.5021368265151978\n",
      "0.9449\n",
      "the 16500 setps AND the loss on the train is:1.481860876083374\n",
      "0.9402\n",
      "the 17000 setps AND the loss on the train is:1.5099526643753052\n",
      "0.9434\n",
      "the 17500 setps AND the loss on the train is:1.5915751457214355\n",
      "0.9425\n",
      "the 18000 setps AND the loss on the train is:1.5360056161880493\n",
      "0.9453\n",
      "the 18500 setps AND the loss on the train is:1.4950740337371826\n",
      "0.9445\n",
      "the 19000 setps AND the loss on the train is:1.5553096532821655\n",
      "0.9422\n",
      "the 19500 setps AND the loss on the train is:1.4602614641189575\n",
      "0.9478\n",
      "the 20000 setps AND the loss on the train is:1.5096971988677979\n",
      "0.9461\n",
      "the 20500 setps AND the loss on the train is:1.544341802597046\n",
      "0.9452\n",
      "the 21000 setps AND the loss on the train is:1.4812136888504028\n",
      "0.9481\n",
      "the 21500 setps AND the loss on the train is:1.5552912950515747\n",
      "0.9444\n",
      "the 22000 setps AND the loss on the train is:1.4327973127365112\n",
      "0.9473\n",
      "the 22500 setps AND the loss on the train is:1.5059113502502441\n",
      "0.9448\n",
      "the 23000 setps AND the loss on the train is:1.4157400131225586\n",
      "0.9469\n",
      "the 23500 setps AND the loss on the train is:1.4932730197906494\n",
      "0.9471\n",
      "the 24000 setps AND the loss on the train is:1.5796875953674316\n",
      "0.9453\n",
      "the 24500 setps AND the loss on the train is:1.4285099506378174\n",
      "0.9478\n",
      "the 25000 setps AND the loss on the train is:1.5002663135528564\n",
      "0.942\n",
      "the 25500 setps AND the loss on the train is:1.492078423500061\n",
      "0.9488\n",
      "the 26000 setps AND the loss on the train is:1.5693894624710083\n",
      "0.9478\n",
      "the 26500 setps AND the loss on the train is:1.4999229907989502\n",
      "0.9451\n",
      "the 27000 setps AND the loss on the train is:1.544198989868164\n",
      "0.9468\n",
      "the 27500 setps AND the loss on the train is:1.4730110168457031\n",
      "0.9489\n",
      "the 28000 setps AND the loss on the train is:1.4767242670059204\n",
      "0.9458\n",
      "the 28500 setps AND the loss on the train is:1.5495003461837769\n",
      "0.9466\n",
      "the 29000 setps AND the loss on the train is:1.454854130744934\n",
      "0.9477\n",
      "the 29500 setps AND the loss on the train is:1.4519304037094116\n",
      "0.9471\n",
      "the 30000 setps AND the loss on the train is:1.4996906518936157\n",
      "0.9478\n",
      "the 30500 setps AND the loss on the train is:1.5041502714157104\n",
      "0.9481\n",
      "the 31000 setps AND the loss on the train is:1.4256350994110107\n",
      "0.9469\n",
      "the 31500 setps AND the loss on the train is:1.5204946994781494\n",
      "0.9491\n",
      "the 32000 setps AND the loss on the train is:1.4240745306015015\n",
      "0.9422\n",
      "the 32500 setps AND the loss on the train is:1.4997444152832031\n",
      "0.9475\n",
      "the 33000 setps AND the loss on the train is:1.4629415273666382\n",
      "0.9467\n",
      "the 33500 setps AND the loss on the train is:1.4657347202301025\n",
      "0.9479\n",
      "the 34000 setps AND the loss on the train is:1.4855620861053467\n",
      "0.9469\n",
      "the 34500 setps AND the loss on the train is:1.525303840637207\n",
      "0.9478\n",
      "the 35000 setps AND the loss on the train is:1.5194122791290283\n",
      "0.9439\n",
      "the 35500 setps AND the loss on the train is:1.562963843345642\n",
      "0.9474\n",
      "the 36000 setps AND the loss on the train is:1.4358750581741333\n",
      "0.9421\n",
      "the 36500 setps AND the loss on the train is:1.4421284198760986\n",
      "0.9479\n",
      "the 37000 setps AND the loss on the train is:1.5864591598510742\n",
      "0.9472\n",
      "the 37500 setps AND the loss on the train is:1.4447773694992065\n",
      "0.9433\n",
      "the 38000 setps AND the loss on the train is:1.4460529088974\n",
      "0.9495\n",
      "the 38500 setps AND the loss on the train is:1.5315313339233398\n",
      "0.9496\n",
      "the 39000 setps AND the loss on the train is:1.4222090244293213\n",
      "0.9508\n",
      "the 39500 setps AND the loss on the train is:1.472101092338562\n",
      "0.9486\n",
      "the 40000 setps AND the loss on the train is:1.6174774169921875\n",
      "0.9504\n",
      "the 40500 setps AND the loss on the train is:1.4276411533355713\n",
      "0.9501\n",
      "the 41000 setps AND the loss on the train is:1.5237325429916382\n",
      "0.9486\n",
      "the 41500 setps AND the loss on the train is:1.419873595237732\n",
      "0.9481\n",
      "the 42000 setps AND the loss on the train is:1.4737099409103394\n",
      "0.9476\n",
      "the 42500 setps AND the loss on the train is:1.499575138092041\n",
      "0.9502\n",
      "the 43000 setps AND the loss on the train is:1.513562560081482\n",
      "0.9487\n",
      "the 43500 setps AND the loss on the train is:1.4606215953826904\n",
      "0.9479\n",
      "the 44000 setps AND the loss on the train is:1.4329252243041992\n",
      "0.9473\n",
      "the 44500 setps AND the loss on the train is:1.5842921733856201\n",
      "0.9486\n",
      "the 45000 setps AND the loss on the train is:1.6924142837524414\n",
      "0.9468\n",
      "the 45500 setps AND the loss on the train is:1.454736351966858\n",
      "0.9496\n",
      "the 46000 setps AND the loss on the train is:1.5849957466125488\n",
      "0.9486\n",
      "the 46500 setps AND the loss on the train is:1.482049822807312\n",
      "0.9482\n",
      "the 47000 setps AND the loss on the train is:1.481999397277832\n",
      "0.9496\n",
      "the 47500 setps AND the loss on the train is:1.4564967155456543\n",
      "0.9485\n",
      "the 48000 setps AND the loss on the train is:1.504915714263916\n",
      "0.9479\n",
      "the 48500 setps AND the loss on the train is:1.467684268951416\n",
      "0.9499\n",
      "the 49000 setps AND the loss on the train is:1.552941918373108\n",
      "0.9501\n",
      "the 49500 setps AND the loss on the train is:1.5105446577072144\n",
      "0.9504\n",
      "the 50000 setps AND the loss on the train is:1.4936774969100952\n",
      "0.9514\n",
      "the 50500 setps AND the loss on the train is:1.5371123552322388\n",
      "0.9495\n",
      "the 51000 setps AND the loss on the train is:1.5330334901809692\n",
      "0.9509\n",
      "the 51500 setps AND the loss on the train is:1.4674477577209473\n",
      "0.9487\n",
      "the 52000 setps AND the loss on the train is:1.4595510959625244\n",
      "0.9488\n",
      "the 52500 setps AND the loss on the train is:1.5577820539474487\n",
      "0.9492\n",
      "the 53000 setps AND the loss on the train is:1.451452374458313\n",
      "0.9498\n",
      "the 53500 setps AND the loss on the train is:1.4661328792572021\n",
      "0.9479\n",
      "the 54000 setps AND the loss on the train is:1.4609503746032715\n",
      "0.9511\n",
      "the 54500 setps AND the loss on the train is:1.4958237409591675\n",
      "0.948\n",
      "the 55000 setps AND the loss on the train is:1.4634976387023926\n",
      "0.9508\n",
      "the 55500 setps AND the loss on the train is:1.4595024585723877\n",
      "0.9508\n",
      "the 56000 setps AND the loss on the train is:1.4935834407806396\n",
      "0.9512\n",
      "the 56500 setps AND the loss on the train is:1.4733333587646484\n",
      "0.9502\n",
      "the 57000 setps AND the loss on the train is:1.4913840293884277\n",
      "0.9479\n",
      "the 57500 setps AND the loss on the train is:1.5454516410827637\n",
      "0.9499\n",
      "the 58000 setps AND the loss on the train is:1.4721137285232544\n",
      "0.9503\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 58500 setps AND the loss on the train is:1.5034269094467163\n",
      "0.9448\n",
      "the 59000 setps AND the loss on the train is:1.430454969406128\n",
      "0.9494\n",
      "the 59500 setps AND the loss on the train is:1.525890827178955\n",
      "0.95\n",
      "the 60000 setps AND the loss on the train is:1.5190534591674805\n",
      "0.9521\n",
      "the 60500 setps AND the loss on the train is:1.4918417930603027\n",
      "0.9498\n",
      "the 61000 setps AND the loss on the train is:1.4300549030303955\n",
      "0.9499\n",
      "the 61500 setps AND the loss on the train is:1.5282572507858276\n",
      "0.9499\n",
      "the 62000 setps AND the loss on the train is:1.4685426950454712\n",
      "0.9499\n",
      "the 62500 setps AND the loss on the train is:1.4403660297393799\n",
      "0.9522\n",
      "the 63000 setps AND the loss on the train is:1.442081093788147\n",
      "0.9502\n",
      "the 63500 setps AND the loss on the train is:1.509660005569458\n",
      "0.9476\n",
      "the 64000 setps AND the loss on the train is:1.4786134958267212\n",
      "0.9497\n",
      "the 64500 setps AND the loss on the train is:1.4969664812088013\n",
      "0.9469\n",
      "the 65000 setps AND the loss on the train is:1.4428465366363525\n",
      "0.9515\n",
      "the 65500 setps AND the loss on the train is:1.550779104232788\n",
      "0.9507\n",
      "the 66000 setps AND the loss on the train is:1.5306835174560547\n",
      "0.9477\n",
      "the 66500 setps AND the loss on the train is:1.4194613695144653\n",
      "0.9503\n",
      "the 67000 setps AND the loss on the train is:1.547921061515808\n",
      "0.952\n",
      "the 67500 setps AND the loss on the train is:1.503701090812683\n",
      "0.947\n",
      "the 68000 setps AND the loss on the train is:1.4781367778778076\n",
      "0.9476\n",
      "the 68500 setps AND the loss on the train is:1.490376591682434\n",
      "0.9504\n",
      "the 69000 setps AND the loss on the train is:1.463199257850647\n",
      "0.9453\n",
      "the 69500 setps AND the loss on the train is:1.4583909511566162\n",
      "0.9502\n",
      "the 70000 setps AND the loss on the train is:1.488632082939148\n",
      "0.9516\n",
      "the 70500 setps AND the loss on the train is:1.4655412435531616\n",
      "0.9515\n",
      "the 71000 setps AND the loss on the train is:1.5051506757736206\n",
      "0.9528\n",
      "the 71500 setps AND the loss on the train is:1.468538761138916\n",
      "0.9512\n",
      "the 72000 setps AND the loss on the train is:1.5247818231582642\n",
      "0.9473\n",
      "the 72500 setps AND the loss on the train is:1.4961060285568237\n",
      "0.9482\n",
      "the 73000 setps AND the loss on the train is:1.4596021175384521\n",
      "0.9513\n",
      "the 73500 setps AND the loss on the train is:1.4879907369613647\n",
      "0.9506\n",
      "the 74000 setps AND the loss on the train is:1.4316778182983398\n",
      "0.9496\n",
      "the 74500 setps AND the loss on the train is:1.586923599243164\n",
      "0.9505\n",
      "the 75000 setps AND the loss on the train is:1.4332311153411865\n",
      "0.9515\n",
      "the 75500 setps AND the loss on the train is:1.4772858619689941\n",
      "0.9503\n",
      "the 76000 setps AND the loss on the train is:1.4735820293426514\n",
      "0.949\n",
      "the 76500 setps AND the loss on the train is:1.4279701709747314\n",
      "0.951\n",
      "the 77000 setps AND the loss on the train is:1.462623953819275\n",
      "0.9521\n",
      "the 77500 setps AND the loss on the train is:1.4697449207305908\n",
      "0.9466\n",
      "the 78000 setps AND the loss on the train is:1.3927664756774902\n",
      "0.9504\n",
      "the 78500 setps AND the loss on the train is:1.4197653532028198\n",
      "0.9511\n",
      "the 79000 setps AND the loss on the train is:1.5377726554870605\n",
      "0.9511\n",
      "the 79500 setps AND the loss on the train is:1.46798837184906\n",
      "0.9518\n",
      "the 80000 setps AND the loss on the train is:1.4527357816696167\n",
      "0.9465\n",
      "the 80500 setps AND the loss on the train is:1.452672004699707\n",
      "0.9509\n",
      "the 81000 setps AND the loss on the train is:1.544470191001892\n",
      "0.9501\n",
      "the 81500 setps AND the loss on the train is:1.4426735639572144\n",
      "0.9504\n",
      "the 82000 setps AND the loss on the train is:1.4436739683151245\n",
      "0.948\n",
      "the 82500 setps AND the loss on the train is:1.457326889038086\n",
      "0.9483\n",
      "the 83000 setps AND the loss on the train is:1.4313457012176514\n",
      "0.9485\n",
      "the 83500 setps AND the loss on the train is:1.4568490982055664\n",
      "0.9476\n",
      "the 84000 setps AND the loss on the train is:1.4990382194519043\n",
      "0.941\n",
      "the 84500 setps AND the loss on the train is:1.4605134725570679\n",
      "0.9524\n",
      "the 85000 setps AND the loss on the train is:1.4105726480484009\n",
      "0.952\n",
      "the 85500 setps AND the loss on the train is:1.4987335205078125\n",
      "0.949\n",
      "the 86000 setps AND the loss on the train is:1.4620177745819092\n",
      "0.9487\n",
      "the 86500 setps AND the loss on the train is:1.4012151956558228\n",
      "0.9526\n",
      "the 87000 setps AND the loss on the train is:1.509169340133667\n",
      "0.947\n",
      "the 87500 setps AND the loss on the train is:1.614040732383728\n",
      "0.9494\n",
      "the 88000 setps AND the loss on the train is:1.4681042432785034\n",
      "0.9455\n",
      "the 88500 setps AND the loss on the train is:1.4709303379058838\n",
      "0.9511\n",
      "the 89000 setps AND the loss on the train is:1.4885369539260864\n",
      "0.9512\n",
      "the 89500 setps AND the loss on the train is:1.5486572980880737\n",
      "0.951\n",
      "the 90000 setps AND the loss on the train is:1.5385637283325195\n",
      "0.9465\n",
      "the 90500 setps AND the loss on the train is:1.562168836593628\n",
      "0.9489\n",
      "the 91000 setps AND the loss on the train is:1.5457359552383423\n",
      "0.9492\n",
      "the 91500 setps AND the loss on the train is:1.4314966201782227\n",
      "0.9484\n",
      "the 92000 setps AND the loss on the train is:1.4490407705307007\n",
      "0.9513\n",
      "the 92500 setps AND the loss on the train is:1.5014384984970093\n",
      "0.9485\n",
      "the 93000 setps AND the loss on the train is:1.4661985635757446\n",
      "0.9494\n",
      "the 93500 setps AND the loss on the train is:1.5116382837295532\n",
      "0.9511\n",
      "the 94000 setps AND the loss on the train is:1.4234215021133423\n",
      "0.9488\n",
      "the 94500 setps AND the loss on the train is:1.4629957675933838\n",
      "0.9481\n",
      "the 95000 setps AND the loss on the train is:1.4426053762435913\n",
      "0.9509\n",
      "the 95500 setps AND the loss on the train is:1.4193581342697144\n",
      "0.9475\n",
      "the 96000 setps AND the loss on the train is:1.5031731128692627\n",
      "0.9495\n",
      "the 96500 setps AND the loss on the train is:1.5230605602264404\n",
      "0.9487\n",
      "the 97000 setps AND the loss on the train is:1.481900691986084\n",
      "0.949\n",
      "the 97500 setps AND the loss on the train is:1.482342004776001\n",
      "0.9492\n",
      "the 98000 setps AND the loss on the train is:1.469656229019165\n",
      "0.9515\n",
      "the 98500 setps AND the loss on the train is:1.527156949043274\n",
      "0.9514\n",
      "the 99000 setps AND the loss on the train is:1.5318348407745361\n",
      "0.9496\n",
      "the 99500 setps AND the loss on the train is:1.4927318096160889\n",
      "0.9509\n",
      "the 100000 setps AND the loss on the train is:1.4990359544754028\n",
      "0.9511\n",
      "the 100500 setps AND the loss on the train is:1.5574262142181396\n",
      "0.9514\n",
      "the 101000 setps AND the loss on the train is:1.4108836650848389\n",
      "0.9498\n",
      "the 101500 setps AND the loss on the train is:1.4566203355789185\n",
      "0.9505\n",
      "the 102000 setps AND the loss on the train is:1.4364750385284424\n",
      "0.9519\n",
      "the 102500 setps AND the loss on the train is:1.496358871459961\n",
      "0.9505\n",
      "the 103000 setps AND the loss on the train is:1.4541188478469849\n",
      "0.9499\n",
      "the 103500 setps AND the loss on the train is:1.5161997079849243\n",
      "0.9547\n",
      "the 104000 setps AND the loss on the train is:1.4533584117889404\n",
      "0.9506\n",
      "the 104500 setps AND the loss on the train is:1.5448355674743652\n",
      "0.9486\n",
      "the 105000 setps AND the loss on the train is:1.5927464962005615\n",
      "0.9513\n",
      "the 105500 setps AND the loss on the train is:1.4340916872024536\n",
      "0.9494\n",
      "the 106000 setps AND the loss on the train is:1.5107083320617676\n",
      "0.9513\n",
      "the 106500 setps AND the loss on the train is:1.4373351335525513\n",
      "0.9519\n",
      "the 107000 setps AND the loss on the train is:1.4636147022247314\n",
      "0.95\n",
      "the 107500 setps AND the loss on the train is:1.5589163303375244\n",
      "0.9509\n",
      "the 108000 setps AND the loss on the train is:1.481619119644165\n",
      "0.9483\n",
      "the 108500 setps AND the loss on the train is:1.4678945541381836\n",
      "0.9506\n",
      "the 109000 setps AND the loss on the train is:1.447414517402649\n",
      "0.9537\n",
      "the 109500 setps AND the loss on the train is:1.5110284090042114\n",
      "0.9502\n",
      "the 110000 setps AND the loss on the train is:1.4668899774551392\n",
      "0.9522\n",
      "the 110500 setps AND the loss on the train is:1.5236437320709229\n",
      "0.9513\n",
      "the 111000 setps AND the loss on the train is:1.6155388355255127\n",
      "0.9497\n",
      "the 111500 setps AND the loss on the train is:1.4202899932861328\n",
      "0.9501\n",
      "the 112000 setps AND the loss on the train is:1.5523258447647095\n",
      "0.9498\n",
      "the 112500 setps AND the loss on the train is:1.450284719467163\n",
      "0.9511\n",
      "the 113000 setps AND the loss on the train is:1.48197340965271\n",
      "0.9486\n",
      "the 113500 setps AND the loss on the train is:1.4592385292053223\n",
      "0.9519\n",
      "the 114000 setps AND the loss on the train is:1.4482394456863403\n",
      "0.9517\n",
      "the 114500 setps AND the loss on the train is:1.4843175411224365\n",
      "0.9508\n",
      "the 115000 setps AND the loss on the train is:1.510411262512207\n",
      "0.95\n",
      "the 115500 setps AND the loss on the train is:1.5087089538574219\n",
      "0.9507\n",
      "the 116000 setps AND the loss on the train is:1.4501807689666748\n",
      "0.9488\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 116500 setps AND the loss on the train is:1.396574854850769\n",
      "0.9499\n",
      "the 117000 setps AND the loss on the train is:1.4680923223495483\n",
      "0.9491\n",
      "the 117500 setps AND the loss on the train is:1.5004205703735352\n",
      "0.9493\n",
      "the 118000 setps AND the loss on the train is:1.4309040307998657\n",
      "0.9512\n",
      "the 118500 setps AND the loss on the train is:1.524695873260498\n",
      "0.9483\n",
      "the 119000 setps AND the loss on the train is:1.5069775581359863\n",
      "0.9514\n",
      "the 119500 setps AND the loss on the train is:1.493547797203064\n",
      "0.9499\n",
      "the 120000 setps AND the loss on the train is:1.4743459224700928\n",
      "0.9453\n",
      "the 120500 setps AND the loss on the train is:1.5388798713684082\n",
      "0.9503\n",
      "the 121000 setps AND the loss on the train is:1.4637746810913086\n",
      "0.9499\n",
      "the 121500 setps AND the loss on the train is:1.4707837104797363\n",
      "0.9471\n",
      "the 122000 setps AND the loss on the train is:1.4571986198425293\n",
      "0.9502\n",
      "the 122500 setps AND the loss on the train is:1.4559533596038818\n",
      "0.9503\n",
      "the 123000 setps AND the loss on the train is:1.5332344770431519\n",
      "0.9506\n",
      "the 123500 setps AND the loss on the train is:1.510411024093628\n",
      "0.9499\n",
      "the 124000 setps AND the loss on the train is:1.5510501861572266\n",
      "0.9495\n",
      "the 124500 setps AND the loss on the train is:1.4751653671264648\n",
      "0.95\n",
      "the 125000 setps AND the loss on the train is:1.6016418933868408\n",
      "0.9501\n",
      "the 125500 setps AND the loss on the train is:1.494919776916504\n",
      "0.9515\n",
      "the 126000 setps AND the loss on the train is:1.5033621788024902\n",
      "0.9519\n",
      "the 126500 setps AND the loss on the train is:1.4909082651138306\n",
      "0.9526\n",
      "the 127000 setps AND the loss on the train is:1.5058273077011108\n",
      "0.9505\n",
      "the 127500 setps AND the loss on the train is:1.6025413274765015\n",
      "0.9482\n",
      "the 128000 setps AND the loss on the train is:1.481019377708435\n",
      "0.9495\n",
      "the 128500 setps AND the loss on the train is:1.5996757745742798\n",
      "0.9521\n",
      "the 129000 setps AND the loss on the train is:1.460073471069336\n",
      "0.9502\n",
      "the 129500 setps AND the loss on the train is:1.4465399980545044\n",
      "0.9519\n",
      "the 130000 setps AND the loss on the train is:1.4802478551864624\n",
      "0.9507\n",
      "the 130500 setps AND the loss on the train is:1.5654151439666748\n",
      "0.9507\n",
      "the 131000 setps AND the loss on the train is:1.444742202758789\n",
      "0.9509\n",
      "the 131500 setps AND the loss on the train is:1.4258795976638794\n",
      "0.9494\n",
      "the 132000 setps AND the loss on the train is:1.4331309795379639\n",
      "0.9475\n",
      "the 132500 setps AND the loss on the train is:1.4189766645431519\n",
      "0.9512\n",
      "the 133000 setps AND the loss on the train is:1.510952353477478\n",
      "0.9493\n",
      "the 133500 setps AND the loss on the train is:1.5142250061035156\n",
      "0.9481\n",
      "the 134000 setps AND the loss on the train is:1.4889354705810547\n",
      "0.9494\n",
      "the 134500 setps AND the loss on the train is:1.5001260042190552\n",
      "0.9498\n",
      "the 135000 setps AND the loss on the train is:1.537196397781372\n",
      "0.9492\n",
      "the 135500 setps AND the loss on the train is:1.4962027072906494\n",
      "0.9504\n",
      "the 136000 setps AND the loss on the train is:1.483262300491333\n",
      "0.9485\n",
      "the 136500 setps AND the loss on the train is:1.547454833984375\n",
      "0.9521\n",
      "the 137000 setps AND the loss on the train is:1.4888333082199097\n",
      "0.9504\n",
      "the 137500 setps AND the loss on the train is:1.4781956672668457\n",
      "0.9471\n",
      "the 138000 setps AND the loss on the train is:1.4413138628005981\n",
      "0.9499\n",
      "the 138500 setps AND the loss on the train is:1.4343340396881104\n",
      "0.9515\n",
      "the 139000 setps AND the loss on the train is:1.4659936428070068\n",
      "0.9493\n",
      "the 139500 setps AND the loss on the train is:1.488508701324463\n",
      "0.9501\n",
      "the 140000 setps AND the loss on the train is:1.4923871755599976\n",
      "0.9488\n",
      "the 140500 setps AND the loss on the train is:1.4329595565795898\n",
      "0.9526\n",
      "the 141000 setps AND the loss on the train is:1.5033760070800781\n",
      "0.9506\n",
      "the 141500 setps AND the loss on the train is:1.4400568008422852\n",
      "0.9507\n",
      "the 142000 setps AND the loss on the train is:1.5097602605819702\n",
      "0.948\n",
      "the 142500 setps AND the loss on the train is:1.4299733638763428\n",
      "0.9503\n",
      "the 143000 setps AND the loss on the train is:1.4786008596420288\n",
      "0.9517\n",
      "the 143500 setps AND the loss on the train is:1.5118824243545532\n",
      "0.9505\n",
      "the 144000 setps AND the loss on the train is:1.5525002479553223\n",
      "0.9446\n",
      "the 144500 setps AND the loss on the train is:1.4714874029159546\n",
      "0.952\n",
      "the 145000 setps AND the loss on the train is:1.4695749282836914\n",
      "0.9522\n",
      "the 145500 setps AND the loss on the train is:1.4387885332107544\n",
      "0.9517\n",
      "the 146000 setps AND the loss on the train is:1.5066845417022705\n",
      "0.9502\n",
      "the 146500 setps AND the loss on the train is:1.5183216333389282\n",
      "0.9503\n",
      "the 147000 setps AND the loss on the train is:1.4818933010101318\n",
      "0.9475\n",
      "the 147500 setps AND the loss on the train is:1.5890724658966064\n",
      "0.9507\n",
      "the 148000 setps AND the loss on the train is:1.4694985151290894\n",
      "0.9523\n",
      "the 148500 setps AND the loss on the train is:1.5698788166046143\n",
      "0.9502\n",
      "the 149000 setps AND the loss on the train is:1.4891413450241089\n",
      "0.9522\n",
      "the 149500 setps AND the loss on the train is:1.5139309167861938\n",
      "0.9483\n",
      "the 150000 setps AND the loss on the train is:1.4442704916000366\n",
      "0.9508\n",
      "the 150500 setps AND the loss on the train is:1.45259690284729\n",
      "0.9524\n",
      "the 151000 setps AND the loss on the train is:1.4464073181152344\n",
      "0.9506\n",
      "the 151500 setps AND the loss on the train is:1.5245991945266724\n",
      "0.9499\n",
      "the 152000 setps AND the loss on the train is:1.5598080158233643\n",
      "0.9485\n",
      "the 152500 setps AND the loss on the train is:1.56894850730896\n",
      "0.9492\n",
      "the 153000 setps AND the loss on the train is:1.4597328901290894\n",
      "0.9495\n",
      "the 153500 setps AND the loss on the train is:1.4939812421798706\n",
      "0.9503\n",
      "the 154000 setps AND the loss on the train is:1.454674243927002\n",
      "0.9486\n",
      "the 154500 setps AND the loss on the train is:1.4519085884094238\n",
      "0.9482\n",
      "the 155000 setps AND the loss on the train is:1.4140591621398926\n",
      "0.9511\n",
      "the 155500 setps AND the loss on the train is:1.615361213684082\n",
      "0.9494\n",
      "the 156000 setps AND the loss on the train is:1.4539062976837158\n",
      "0.9515\n",
      "the 156500 setps AND the loss on the train is:1.5835448503494263\n",
      "0.9487\n",
      "the 157000 setps AND the loss on the train is:1.4757684469223022\n",
      "0.9479\n",
      "the 157500 setps AND the loss on the train is:1.4698156118392944\n",
      "0.9512\n",
      "the 158000 setps AND the loss on the train is:1.4355075359344482\n",
      "0.9507\n",
      "the 158500 setps AND the loss on the train is:1.4763351678848267\n",
      "0.9498\n",
      "the 159000 setps AND the loss on the train is:1.472099781036377\n",
      "0.9472\n",
      "the 159500 setps AND the loss on the train is:1.4894330501556396\n",
      "0.9522\n",
      "the 160000 setps AND the loss on the train is:1.463019847869873\n",
      "0.951\n",
      "the 160500 setps AND the loss on the train is:1.476679801940918\n",
      "0.9506\n",
      "the 161000 setps AND the loss on the train is:1.4242873191833496\n",
      "0.9504\n",
      "the 161500 setps AND the loss on the train is:1.4522536993026733\n",
      "0.9525\n",
      "the 162000 setps AND the loss on the train is:1.5104212760925293\n",
      "0.9512\n",
      "the 162500 setps AND the loss on the train is:1.5063532590866089\n",
      "0.9503\n",
      "the 163000 setps AND the loss on the train is:1.466738224029541\n",
      "0.951\n",
      "the 163500 setps AND the loss on the train is:1.4600245952606201\n",
      "0.9508\n",
      "the 164000 setps AND the loss on the train is:1.4843555688858032\n",
      "0.9458\n",
      "the 164500 setps AND the loss on the train is:1.5030241012573242\n",
      "0.9503\n",
      "the 165000 setps AND the loss on the train is:1.4482054710388184\n",
      "0.9488\n",
      "the 165500 setps AND the loss on the train is:1.475322961807251\n",
      "0.9487\n",
      "the 166000 setps AND the loss on the train is:1.4202500581741333\n",
      "0.9521\n",
      "the 166500 setps AND the loss on the train is:1.4885367155075073\n",
      "0.9528\n",
      "the 167000 setps AND the loss on the train is:1.5083394050598145\n",
      "0.9522\n",
      "the 167500 setps AND the loss on the train is:1.4799621105194092\n",
      "0.9476\n",
      "the 168000 setps AND the loss on the train is:1.474110722541809\n",
      "0.9513\n",
      "the 168500 setps AND the loss on the train is:1.4904369115829468\n",
      "0.95\n",
      "the 169000 setps AND the loss on the train is:1.4177587032318115\n",
      "0.9523\n",
      "the 169500 setps AND the loss on the train is:1.5252004861831665\n",
      "0.9497\n",
      "the 170000 setps AND the loss on the train is:1.4633039236068726\n",
      "0.9492\n",
      "the 170500 setps AND the loss on the train is:1.4305932521820068\n",
      "0.9492\n",
      "the 171000 setps AND the loss on the train is:1.5338568687438965\n",
      "0.9505\n",
      "the 171500 setps AND the loss on the train is:1.4582557678222656\n",
      "0.9492\n",
      "the 172000 setps AND the loss on the train is:1.4386825561523438\n",
      "0.9516\n",
      "the 172500 setps AND the loss on the train is:1.466554880142212\n",
      "0.9514\n",
      "the 173000 setps AND the loss on the train is:1.4771636724472046\n",
      "0.9502\n",
      "the 173500 setps AND the loss on the train is:1.4550654888153076\n",
      "0.9496\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 174000 setps AND the loss on the train is:1.4233427047729492\n",
      "0.9523\n",
      "the 174500 setps AND the loss on the train is:1.5020568370819092\n",
      "0.9475\n",
      "the 175000 setps AND the loss on the train is:1.5598973035812378\n",
      "0.9505\n",
      "the 175500 setps AND the loss on the train is:1.5079598426818848\n",
      "0.9491\n",
      "the 176000 setps AND the loss on the train is:1.4263395071029663\n",
      "0.9506\n",
      "the 176500 setps AND the loss on the train is:1.416148066520691\n",
      "0.9513\n",
      "the 177000 setps AND the loss on the train is:1.4543462991714478\n",
      "0.9518\n",
      "the 177500 setps AND the loss on the train is:1.4532861709594727\n",
      "0.9509\n",
      "the 178000 setps AND the loss on the train is:1.4751418828964233\n",
      "0.9509\n",
      "the 178500 setps AND the loss on the train is:1.4509788751602173\n",
      "0.9475\n",
      "the 179000 setps AND the loss on the train is:1.4480204582214355\n",
      "0.9526\n",
      "the 179500 setps AND the loss on the train is:1.471435546875\n",
      "0.9506\n",
      "the 180000 setps AND the loss on the train is:1.5290241241455078\n",
      "0.9519\n",
      "the 180500 setps AND the loss on the train is:1.4499887228012085\n",
      "0.9513\n",
      "the 181000 setps AND the loss on the train is:1.5131428241729736\n",
      "0.9505\n",
      "the 181500 setps AND the loss on the train is:1.5444467067718506\n",
      "0.9513\n",
      "the 182000 setps AND the loss on the train is:1.4172333478927612\n",
      "0.9475\n",
      "the 182500 setps AND the loss on the train is:1.4828040599822998\n",
      "0.9514\n",
      "the 183000 setps AND the loss on the train is:1.5024586915969849\n",
      "0.9487\n",
      "the 183500 setps AND the loss on the train is:1.471028447151184\n",
      "0.9502\n",
      "the 184000 setps AND the loss on the train is:1.4390006065368652\n",
      "0.9508\n",
      "the 184500 setps AND the loss on the train is:1.502227544784546\n",
      "0.9514\n",
      "the 185000 setps AND the loss on the train is:1.4992331266403198\n",
      "0.95\n",
      "the 185500 setps AND the loss on the train is:1.4840441942214966\n",
      "0.9512\n",
      "the 186000 setps AND the loss on the train is:1.461059808731079\n",
      "0.9483\n",
      "the 186500 setps AND the loss on the train is:1.4294945001602173\n",
      "0.9514\n",
      "the 187000 setps AND the loss on the train is:1.4363492727279663\n",
      "0.9505\n",
      "the 187500 setps AND the loss on the train is:1.4414317607879639\n",
      "0.9512\n",
      "the 188000 setps AND the loss on the train is:1.452370047569275\n",
      "0.9512\n",
      "the 188500 setps AND the loss on the train is:1.5254261493682861\n",
      "0.9523\n",
      "the 189000 setps AND the loss on the train is:1.4121612310409546\n",
      "0.9501\n",
      "the 189500 setps AND the loss on the train is:1.4310908317565918\n",
      "0.9502\n",
      "the 190000 setps AND the loss on the train is:1.4597755670547485\n",
      "0.9508\n",
      "the 190500 setps AND the loss on the train is:1.5015017986297607\n",
      "0.9511\n",
      "the 191000 setps AND the loss on the train is:1.407931923866272\n",
      "0.9505\n",
      "the 191500 setps AND the loss on the train is:1.524849534034729\n",
      "0.9504\n",
      "the 192000 setps AND the loss on the train is:1.5072450637817383\n",
      "0.9519\n",
      "the 192500 setps AND the loss on the train is:1.446974277496338\n",
      "0.9523\n",
      "the 193000 setps AND the loss on the train is:1.5025231838226318\n",
      "0.9488\n",
      "the 193500 setps AND the loss on the train is:1.5077054500579834\n",
      "0.9491\n",
      "the 194000 setps AND the loss on the train is:1.461125373840332\n",
      "0.9482\n",
      "the 194500 setps AND the loss on the train is:1.4998692274093628\n",
      "0.9487\n",
      "the 195000 setps AND the loss on the train is:1.4127371311187744\n",
      "0.9517\n",
      "the 195500 setps AND the loss on the train is:1.4428735971450806\n",
      "0.9492\n",
      "the 196000 setps AND the loss on the train is:1.4254601001739502\n",
      "0.9503\n",
      "the 196500 setps AND the loss on the train is:1.527974247932434\n",
      "0.9522\n",
      "the 197000 setps AND the loss on the train is:1.4603736400604248\n",
      "0.9514\n",
      "the 197500 setps AND the loss on the train is:1.4202650785446167\n",
      "0.9516\n",
      "the 198000 setps AND the loss on the train is:1.4916183948516846\n",
      "0.9512\n",
      "the 198500 setps AND the loss on the train is:1.4639644622802734\n",
      "0.9504\n",
      "the 199000 setps AND the loss on the train is:1.4600822925567627\n",
      "0.9512\n",
      "the 199500 setps AND the loss on the train is:1.520202398300171\n",
      "0.9487\n",
      "the 200000 setps AND the loss on the train is:1.4578120708465576\n",
      "0.9505\n",
      "the 200500 setps AND the loss on the train is:1.4688549041748047\n",
      "0.9521\n",
      "the 201000 setps AND the loss on the train is:1.4605638980865479\n",
      "0.95\n",
      "the 201500 setps AND the loss on the train is:1.4211992025375366\n",
      "0.9509\n",
      "the 202000 setps AND the loss on the train is:1.6008260250091553\n",
      "0.9522\n",
      "the 202500 setps AND the loss on the train is:1.5197529792785645\n",
      "0.9538\n",
      "the 203000 setps AND the loss on the train is:1.5058852434158325\n",
      "0.9526\n",
      "the 203500 setps AND the loss on the train is:1.5043892860412598\n",
      "0.9499\n",
      "the 204000 setps AND the loss on the train is:1.5002694129943848\n",
      "0.95\n",
      "the 204500 setps AND the loss on the train is:1.4711825847625732\n",
      "0.9515\n",
      "the 205000 setps AND the loss on the train is:1.4787406921386719\n",
      "0.95\n",
      "the 205500 setps AND the loss on the train is:1.4348583221435547\n",
      "0.95\n",
      "the 206000 setps AND the loss on the train is:1.5523629188537598\n",
      "0.9508\n",
      "the 206500 setps AND the loss on the train is:1.4451439380645752\n",
      "0.9513\n",
      "the 207000 setps AND the loss on the train is:1.404546856880188\n",
      "0.9504\n",
      "the 207500 setps AND the loss on the train is:1.4851174354553223\n",
      "0.9514\n",
      "the 208000 setps AND the loss on the train is:1.4308754205703735\n",
      "0.9492\n",
      "the 208500 setps AND the loss on the train is:1.4515485763549805\n",
      "0.9495\n",
      "the 209000 setps AND the loss on the train is:1.434077262878418\n",
      "0.9442\n",
      "the 209500 setps AND the loss on the train is:1.4339289665222168\n",
      "0.9511\n",
      "the 210000 setps AND the loss on the train is:1.5183703899383545\n",
      "0.9505\n",
      "the 210500 setps AND the loss on the train is:1.4861544370651245\n",
      "0.9497\n",
      "the 211000 setps AND the loss on the train is:1.4460078477859497\n",
      "0.9511\n",
      "the 211500 setps AND the loss on the train is:1.5437487363815308\n",
      "0.9533\n",
      "the 212000 setps AND the loss on the train is:1.5156540870666504\n",
      "0.9486\n",
      "the 212500 setps AND the loss on the train is:1.4868308305740356\n",
      "0.9548\n",
      "the 213000 setps AND the loss on the train is:1.4429856538772583\n",
      "0.9528\n",
      "the 213500 setps AND the loss on the train is:1.4386202096939087\n",
      "0.9508\n",
      "the 214000 setps AND the loss on the train is:1.5223270654678345\n",
      "0.9534\n",
      "the 214500 setps AND the loss on the train is:1.5456061363220215\n",
      "0.95\n",
      "the 215000 setps AND the loss on the train is:1.5151594877243042\n",
      "0.9496\n",
      "the 215500 setps AND the loss on the train is:1.470001459121704\n",
      "0.95\n",
      "the 216000 setps AND the loss on the train is:1.462712287902832\n",
      "0.9496\n",
      "the 216500 setps AND the loss on the train is:1.4143720865249634\n",
      "0.9513\n",
      "the 217000 setps AND the loss on the train is:1.505049467086792\n",
      "0.9502\n",
      "the 217500 setps AND the loss on the train is:1.495017647743225\n",
      "0.9482\n",
      "the 218000 setps AND the loss on the train is:1.4876083135604858\n",
      "0.9501\n",
      "the 218500 setps AND the loss on the train is:1.519183874130249\n",
      "0.9505\n",
      "the 219000 setps AND the loss on the train is:1.4834939241409302\n",
      "0.9498\n",
      "the 219500 setps AND the loss on the train is:1.535013198852539\n",
      "0.9498\n",
      "the 220000 setps AND the loss on the train is:1.4663994312286377\n",
      "0.9488\n",
      "the 220500 setps AND the loss on the train is:1.4431450366973877\n",
      "0.9513\n",
      "the 221000 setps AND the loss on the train is:1.427183985710144\n",
      "0.9506\n",
      "the 221500 setps AND the loss on the train is:1.4270832538604736\n",
      "0.9508\n",
      "the 222000 setps AND the loss on the train is:1.5685391426086426\n",
      "0.9516\n",
      "the 222500 setps AND the loss on the train is:1.5231945514678955\n",
      "0.9494\n",
      "the 223000 setps AND the loss on the train is:1.502439260482788\n",
      "0.9517\n",
      "the 223500 setps AND the loss on the train is:1.4780876636505127\n",
      "0.9482\n",
      "the 224000 setps AND the loss on the train is:1.479636311531067\n",
      "0.9526\n",
      "the 224500 setps AND the loss on the train is:1.490039587020874\n",
      "0.9524\n",
      "the 225000 setps AND the loss on the train is:1.410116195678711\n",
      "0.9507\n",
      "the 225500 setps AND the loss on the train is:1.4485445022583008\n",
      "0.9517\n",
      "the 226000 setps AND the loss on the train is:1.4552927017211914\n",
      "0.9492\n",
      "the 226500 setps AND the loss on the train is:1.4714698791503906\n",
      "0.9507\n",
      "the 227000 setps AND the loss on the train is:1.4518572092056274\n",
      "0.9487\n",
      "the 227500 setps AND the loss on the train is:1.5451900959014893\n",
      "0.9472\n",
      "the 228000 setps AND the loss on the train is:1.4988226890563965\n",
      "0.9504\n",
      "the 228500 setps AND the loss on the train is:1.4855754375457764\n",
      "0.9503\n",
      "the 229000 setps AND the loss on the train is:1.469846487045288\n",
      "0.9494\n",
      "the 229500 setps AND the loss on the train is:1.4921462535858154\n",
      "0.9493\n",
      "the 230000 setps AND the loss on the train is:1.4078730344772339\n",
      "0.949\n",
      "the 230500 setps AND the loss on the train is:1.521482229232788\n",
      "0.9478\n",
      "the 231000 setps AND the loss on the train is:1.502937912940979\n",
      "0.9516\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 231500 setps AND the loss on the train is:1.4923076629638672\n",
      "0.952\n",
      "the 232000 setps AND the loss on the train is:1.5650146007537842\n",
      "0.9499\n",
      "the 232500 setps AND the loss on the train is:1.475555658340454\n",
      "0.9501\n",
      "the 233000 setps AND the loss on the train is:1.4826043844223022\n",
      "0.9473\n",
      "the 233500 setps AND the loss on the train is:1.5126748085021973\n",
      "0.9508\n",
      "the 234000 setps AND the loss on the train is:1.5300462245941162\n",
      "0.9518\n",
      "the 234500 setps AND the loss on the train is:1.4870229959487915\n",
      "0.9496\n",
      "the 235000 setps AND the loss on the train is:1.4930845499038696\n",
      "0.9515\n",
      "the 235500 setps AND the loss on the train is:1.5489223003387451\n",
      "0.9489\n",
      "the 236000 setps AND the loss on the train is:1.5010566711425781\n",
      "0.9469\n",
      "the 236500 setps AND the loss on the train is:1.4173176288604736\n",
      "0.9509\n",
      "the 237000 setps AND the loss on the train is:1.448851466178894\n",
      "0.9508\n",
      "the 237500 setps AND the loss on the train is:1.5273044109344482\n",
      "0.95\n",
      "the 238000 setps AND the loss on the train is:1.4689065217971802\n",
      "0.9505\n",
      "the 238500 setps AND the loss on the train is:1.4861177206039429\n",
      "0.95\n",
      "the 239000 setps AND the loss on the train is:1.4449063539505005\n",
      "0.9511\n",
      "the 239500 setps AND the loss on the train is:1.4134495258331299\n",
      "0.9537\n",
      "the 240000 setps AND the loss on the train is:1.4842684268951416\n",
      "0.9496\n",
      "the 240500 setps AND the loss on the train is:1.4754849672317505\n",
      "0.9519\n",
      "the 241000 setps AND the loss on the train is:1.4692373275756836\n",
      "0.9499\n",
      "the 241500 setps AND the loss on the train is:1.4826325178146362\n",
      "0.9499\n",
      "the 242000 setps AND the loss on the train is:1.5103949308395386\n",
      "0.9493\n",
      "the 242500 setps AND the loss on the train is:1.5401616096496582\n",
      "0.9527\n",
      "the 243000 setps AND the loss on the train is:1.4901527166366577\n",
      "0.9514\n",
      "the 243500 setps AND the loss on the train is:1.4206821918487549\n",
      "0.9488\n",
      "the 244000 setps AND the loss on the train is:1.4509451389312744\n",
      "0.9526\n",
      "the 244500 setps AND the loss on the train is:1.515270471572876\n",
      "0.9513\n",
      "the 245000 setps AND the loss on the train is:1.5339683294296265\n",
      "0.9488\n",
      "the 245500 setps AND the loss on the train is:1.477518916130066\n",
      "0.9509\n",
      "the 246000 setps AND the loss on the train is:1.4396556615829468\n",
      "0.952\n",
      "the 246500 setps AND the loss on the train is:1.4909616708755493\n",
      "0.9495\n",
      "the 247000 setps AND the loss on the train is:1.5063061714172363\n",
      "0.9499\n",
      "the 247500 setps AND the loss on the train is:1.468407154083252\n",
      "0.9517\n",
      "the 248000 setps AND the loss on the train is:1.4429460763931274\n",
      "0.952\n",
      "the 248500 setps AND the loss on the train is:1.4163620471954346\n",
      "0.9491\n",
      "the 249000 setps AND the loss on the train is:1.5224170684814453\n",
      "0.9511\n",
      "the 249500 setps AND the loss on the train is:1.4490457773208618\n",
      "0.9489\n",
      "the 250000 setps AND the loss on the train is:1.446731448173523\n",
      "0.95\n",
      "the 250500 setps AND the loss on the train is:1.5033823251724243\n",
      "0.9507\n",
      "the 251000 setps AND the loss on the train is:1.487652063369751\n",
      "0.9494\n",
      "the 251500 setps AND the loss on the train is:1.4427530765533447\n",
      "0.9488\n",
      "the 252000 setps AND the loss on the train is:1.4582964181900024\n",
      "0.9486\n",
      "the 252500 setps AND the loss on the train is:1.457912802696228\n",
      "0.951\n",
      "the 253000 setps AND the loss on the train is:1.5824356079101562\n",
      "0.9515\n",
      "the 253500 setps AND the loss on the train is:1.4128273725509644\n",
      "0.952\n",
      "the 254000 setps AND the loss on the train is:1.4693470001220703\n",
      "0.9496\n",
      "the 254500 setps AND the loss on the train is:1.4262466430664062\n",
      "0.9513\n",
      "the 255000 setps AND the loss on the train is:1.4728055000305176\n",
      "0.9512\n",
      "the 255500 setps AND the loss on the train is:1.4862719774246216\n",
      "0.9499\n",
      "the 256000 setps AND the loss on the train is:1.5472131967544556\n",
      "0.9514\n",
      "the 256500 setps AND the loss on the train is:1.4405035972595215\n",
      "0.9509\n",
      "the 257000 setps AND the loss on the train is:1.4737231731414795\n",
      "0.9473\n",
      "the 257500 setps AND the loss on the train is:1.4468722343444824\n",
      "0.9511\n",
      "the 258000 setps AND the loss on the train is:1.507007122039795\n",
      "0.9505\n",
      "the 258500 setps AND the loss on the train is:1.5144261121749878\n",
      "0.9491\n",
      "the 259000 setps AND the loss on the train is:1.4900357723236084\n",
      "0.9516\n",
      "the 259500 setps AND the loss on the train is:1.5311388969421387\n",
      "0.9496\n",
      "the 260000 setps AND the loss on the train is:1.5175752639770508\n",
      "0.9527\n",
      "the 260500 setps AND the loss on the train is:1.5679254531860352\n",
      "0.9533\n",
      "the 261000 setps AND the loss on the train is:1.5541936159133911\n",
      "0.9504\n",
      "the 261500 setps AND the loss on the train is:1.438101053237915\n",
      "0.9512\n",
      "the 262000 setps AND the loss on the train is:1.521505355834961\n",
      "0.9516\n",
      "the 262500 setps AND the loss on the train is:1.4992547035217285\n",
      "0.9507\n",
      "the 263000 setps AND the loss on the train is:1.4531633853912354\n",
      "0.9515\n",
      "the 263500 setps AND the loss on the train is:1.522405982017517\n",
      "0.9521\n",
      "the 264000 setps AND the loss on the train is:1.43287193775177\n",
      "0.9533\n",
      "the 264500 setps AND the loss on the train is:1.4547237157821655\n",
      "0.9508\n",
      "the 265000 setps AND the loss on the train is:1.4860103130340576\n",
      "0.952\n",
      "the 265500 setps AND the loss on the train is:1.4458063840866089\n",
      "0.9509\n",
      "the 266000 setps AND the loss on the train is:1.5018224716186523\n",
      "0.9515\n",
      "the 266500 setps AND the loss on the train is:1.5181167125701904\n",
      "0.9487\n",
      "the 267000 setps AND the loss on the train is:1.5317190885543823\n",
      "0.9485\n",
      "the 267500 setps AND the loss on the train is:1.4666131734848022\n",
      "0.9512\n",
      "the 268000 setps AND the loss on the train is:1.4545344114303589\n",
      "0.9511\n",
      "the 268500 setps AND the loss on the train is:1.5223398208618164\n",
      "0.9519\n",
      "the 269000 setps AND the loss on the train is:1.4155155420303345\n",
      "0.9481\n",
      "the 269500 setps AND the loss on the train is:1.4702260494232178\n",
      "0.948\n",
      "the 270000 setps AND the loss on the train is:1.4797590970993042\n",
      "0.9508\n",
      "the 270500 setps AND the loss on the train is:1.5455944538116455\n",
      "0.9528\n",
      "the 271000 setps AND the loss on the train is:1.4640181064605713\n",
      "0.9495\n",
      "the 271500 setps AND the loss on the train is:1.4433739185333252\n",
      "0.9515\n",
      "the 272000 setps AND the loss on the train is:1.485059142112732\n",
      "0.9537\n",
      "the 272500 setps AND the loss on the train is:1.488800287246704\n",
      "0.9504\n",
      "the 273000 setps AND the loss on the train is:1.5249422788619995\n",
      "0.9507\n",
      "the 273500 setps AND the loss on the train is:1.4287265539169312\n",
      "0.9507\n",
      "the 274000 setps AND the loss on the train is:1.529111385345459\n",
      "0.9458\n",
      "the 274500 setps AND the loss on the train is:1.5072026252746582\n",
      "0.9513\n",
      "the 275000 setps AND the loss on the train is:1.4718905687332153\n",
      "0.9502\n",
      "the 275500 setps AND the loss on the train is:1.5054057836532593\n",
      "0.95\n",
      "the 276000 setps AND the loss on the train is:1.494238257408142\n",
      "0.9484\n",
      "the 276500 setps AND the loss on the train is:1.4558794498443604\n",
      "0.9522\n",
      "the 277000 setps AND the loss on the train is:1.5099023580551147\n",
      "0.9529\n",
      "the 277500 setps AND the loss on the train is:1.4401122331619263\n",
      "0.9516\n",
      "the 278000 setps AND the loss on the train is:1.4756778478622437\n",
      "0.9527\n",
      "the 278500 setps AND the loss on the train is:1.453975796699524\n",
      "0.9498\n",
      "the 279000 setps AND the loss on the train is:1.473457932472229\n",
      "0.9502\n",
      "the 279500 setps AND the loss on the train is:1.5279173851013184\n",
      "0.9523\n",
      "the 280000 setps AND the loss on the train is:1.5407899618148804\n",
      "0.952\n",
      "the 280500 setps AND the loss on the train is:1.449342966079712\n",
      "0.9528\n",
      "the 281000 setps AND the loss on the train is:1.5246543884277344\n",
      "0.9492\n",
      "the 281500 setps AND the loss on the train is:1.4996771812438965\n",
      "0.9499\n",
      "the 282000 setps AND the loss on the train is:1.5171501636505127\n",
      "0.9514\n",
      "the 282500 setps AND the loss on the train is:1.431390643119812\n",
      "0.9493\n",
      "the 283000 setps AND the loss on the train is:1.479326605796814\n",
      "0.9493\n",
      "the 283500 setps AND the loss on the train is:1.4707087278366089\n",
      "0.9492\n",
      "the 284000 setps AND the loss on the train is:1.48969304561615\n",
      "0.9492\n",
      "the 284500 setps AND the loss on the train is:1.4626537561416626\n",
      "0.9529\n",
      "the 285000 setps AND the loss on the train is:1.4516748189926147\n",
      "0.9527\n",
      "the 285500 setps AND the loss on the train is:1.4535577297210693\n",
      "0.9514\n",
      "the 286000 setps AND the loss on the train is:1.4825286865234375\n",
      "0.9524\n",
      "the 286500 setps AND the loss on the train is:1.5557231903076172\n",
      "0.952\n",
      "the 287000 setps AND the loss on the train is:1.431125283241272\n",
      "0.9489\n",
      "the 287500 setps AND the loss on the train is:1.5445064306259155\n",
      "0.9483\n",
      "the 288000 setps AND the loss on the train is:1.4674409627914429\n",
      "0.951\n",
      "the 288500 setps AND the loss on the train is:1.5292129516601562\n",
      "0.9512\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 289000 setps AND the loss on the train is:1.4131850004196167\n",
      "0.9497\n",
      "the 289500 setps AND the loss on the train is:1.5040217638015747\n",
      "0.9542\n",
      "the 290000 setps AND the loss on the train is:1.416113257408142\n",
      "0.9504\n",
      "the 290500 setps AND the loss on the train is:1.4819819927215576\n",
      "0.9513\n",
      "the 291000 setps AND the loss on the train is:1.5293635129928589\n",
      "0.9507\n",
      "the 291500 setps AND the loss on the train is:1.4669711589813232\n",
      "0.9502\n",
      "the 292000 setps AND the loss on the train is:1.4615925550460815\n",
      "0.9513\n",
      "the 292500 setps AND the loss on the train is:1.4424065351486206\n",
      "0.9486\n",
      "the 293000 setps AND the loss on the train is:1.5204969644546509\n",
      "0.9442\n",
      "the 293500 setps AND the loss on the train is:1.4321072101593018\n",
      "0.9506\n",
      "the 294000 setps AND the loss on the train is:1.4770373106002808\n",
      "0.9502\n",
      "the 294500 setps AND the loss on the train is:1.4436770677566528\n",
      "0.9509\n",
      "the 295000 setps AND the loss on the train is:1.4315557479858398\n",
      "0.95\n",
      "the 295500 setps AND the loss on the train is:1.5789777040481567\n",
      "0.9477\n",
      "the 296000 setps AND the loss on the train is:1.5647706985473633\n",
      "0.9488\n",
      "the 296500 setps AND the loss on the train is:1.455100178718567\n",
      "0.9504\n",
      "the 297000 setps AND the loss on the train is:1.495041012763977\n",
      "0.951\n",
      "the 297500 setps AND the loss on the train is:1.457779884338379\n",
      "0.9517\n",
      "the 298000 setps AND the loss on the train is:1.4545341730117798\n",
      "0.9526\n",
      "the 298500 setps AND the loss on the train is:1.4732470512390137\n",
      "0.9488\n",
      "the 299000 setps AND the loss on the train is:1.495073914527893\n",
      "0.9493\n",
      "the 299500 setps AND the loss on the train is:1.5127991437911987\n",
      "0.9525\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,50],0.01)\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=get_weight([50,50],0.01)\n",
    "b2=tf.Variable(tf.random_normal([50]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.relu(logits2)\n",
    "\n",
    "w3=get_weight([50,10],0.01)\n",
    "b3=tf.Variable(tf.random_normal([10]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "\n",
    "global_step=tf.Variable(0,trainable=False)\n",
    "learning_rate=tf.train.exponential_decay(0.1,global_step,500,0.99)\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits3))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits3,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(300000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4.3使用2个隐层,学习率使用指数损失，两层神经元使用tanh激活函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:7348.927734375\n",
      "0.1045\n",
      "the 500 setps AND the loss on the train is:2646.323974609375\n",
      "0.7439\n",
      "the 1000 setps AND the loss on the train is:941.0499267578125\n",
      "0.8573\n",
      "the 1500 setps AND the loss on the train is:339.7421569824219\n",
      "0.8992\n",
      "the 2000 setps AND the loss on the train is:120.33460235595703\n",
      "0.9169\n",
      "the 2500 setps AND the loss on the train is:42.37114334106445\n",
      "0.9236\n",
      "the 3000 setps AND the loss on the train is:15.742847442626953\n",
      "0.9242\n",
      "the 3500 setps AND the loss on the train is:6.600320339202881\n",
      "0.9304\n",
      "the 4000 setps AND the loss on the train is:3.4831292629241943\n",
      "0.9317\n",
      "the 4500 setps AND the loss on the train is:2.305577278137207\n",
      "0.9295\n",
      "the 5000 setps AND the loss on the train is:1.7706178426742554\n",
      "0.9326\n",
      "the 5500 setps AND the loss on the train is:1.6120564937591553\n",
      "0.9298\n",
      "the 6000 setps AND the loss on the train is:1.6125355958938599\n",
      "0.9329\n",
      "the 6500 setps AND the loss on the train is:1.5829589366912842\n",
      "0.9337\n",
      "the 7000 setps AND the loss on the train is:1.5587472915649414\n",
      "0.9361\n",
      "the 7500 setps AND the loss on the train is:1.4546704292297363\n",
      "0.936\n",
      "the 8000 setps AND the loss on the train is:1.648219347000122\n",
      "0.9359\n",
      "the 8500 setps AND the loss on the train is:1.5971946716308594\n",
      "0.9383\n",
      "the 9000 setps AND the loss on the train is:1.588780403137207\n",
      "0.9368\n",
      "the 9500 setps AND the loss on the train is:1.5686516761779785\n",
      "0.9378\n",
      "the 10000 setps AND the loss on the train is:1.608340859413147\n",
      "0.9389\n",
      "the 10500 setps AND the loss on the train is:1.5823876857757568\n",
      "0.9377\n",
      "the 11000 setps AND the loss on the train is:1.4843826293945312\n",
      "0.9396\n",
      "the 11500 setps AND the loss on the train is:1.5493677854537964\n",
      "0.9382\n",
      "the 12000 setps AND the loss on the train is:1.4825721979141235\n",
      "0.9319\n",
      "the 12500 setps AND the loss on the train is:1.5089489221572876\n",
      "0.9394\n",
      "the 13000 setps AND the loss on the train is:1.562149167060852\n",
      "0.9383\n",
      "the 13500 setps AND the loss on the train is:1.5081406831741333\n",
      "0.937\n",
      "the 14000 setps AND the loss on the train is:1.5667049884796143\n",
      "0.9396\n",
      "the 14500 setps AND the loss on the train is:1.5422449111938477\n",
      "0.9384\n",
      "the 15000 setps AND the loss on the train is:1.5172531604766846\n",
      "0.9393\n",
      "the 15500 setps AND the loss on the train is:1.6542110443115234\n",
      "0.9409\n",
      "the 16000 setps AND the loss on the train is:1.5257723331451416\n",
      "0.939\n",
      "the 16500 setps AND the loss on the train is:1.5765159130096436\n",
      "0.9358\n",
      "the 17000 setps AND the loss on the train is:1.5018751621246338\n",
      "0.937\n",
      "the 17500 setps AND the loss on the train is:1.577211856842041\n",
      "0.9409\n",
      "the 18000 setps AND the loss on the train is:1.5545562505722046\n",
      "0.9387\n",
      "the 18500 setps AND the loss on the train is:1.5323834419250488\n",
      "0.9398\n",
      "the 19000 setps AND the loss on the train is:1.5906368494033813\n",
      "0.9374\n",
      "the 19500 setps AND the loss on the train is:1.5683742761611938\n",
      "0.9419\n",
      "the 20000 setps AND the loss on the train is:1.5109666585922241\n",
      "0.9391\n",
      "the 20500 setps AND the loss on the train is:1.5674787759780884\n",
      "0.939\n",
      "the 21000 setps AND the loss on the train is:1.5811619758605957\n",
      "0.9395\n",
      "the 21500 setps AND the loss on the train is:1.5030772686004639\n",
      "0.9416\n",
      "the 22000 setps AND the loss on the train is:1.4943920373916626\n",
      "0.9402\n",
      "the 22500 setps AND the loss on the train is:1.5173696279525757\n",
      "0.942\n",
      "the 23000 setps AND the loss on the train is:1.5245397090911865\n",
      "0.9373\n",
      "the 23500 setps AND the loss on the train is:1.614176630973816\n",
      "0.939\n",
      "the 24000 setps AND the loss on the train is:1.5834280252456665\n",
      "0.9407\n",
      "the 24500 setps AND the loss on the train is:1.542927622795105\n",
      "0.9399\n",
      "the 25000 setps AND the loss on the train is:1.5485743284225464\n",
      "0.9387\n",
      "the 25500 setps AND the loss on the train is:1.5604934692382812\n",
      "0.9349\n",
      "the 26000 setps AND the loss on the train is:1.6159257888793945\n",
      "0.9385\n",
      "the 26500 setps AND the loss on the train is:1.5962239503860474\n",
      "0.9419\n",
      "the 27000 setps AND the loss on the train is:1.5199333429336548\n",
      "0.9421\n",
      "the 27500 setps AND the loss on the train is:1.5684435367584229\n",
      "0.9399\n",
      "the 28000 setps AND the loss on the train is:1.581315279006958\n",
      "0.9397\n",
      "the 28500 setps AND the loss on the train is:1.5866258144378662\n",
      "0.9378\n",
      "the 29000 setps AND the loss on the train is:1.5115104913711548\n",
      "0.9403\n",
      "the 29500 setps AND the loss on the train is:1.5264968872070312\n",
      "0.9394\n",
      "the 30000 setps AND the loss on the train is:1.5510475635528564\n",
      "0.94\n",
      "the 30500 setps AND the loss on the train is:1.588881254196167\n",
      "0.9382\n",
      "the 31000 setps AND the loss on the train is:1.6302242279052734\n",
      "0.9364\n",
      "the 31500 setps AND the loss on the train is:1.5618925094604492\n",
      "0.9367\n",
      "the 32000 setps AND the loss on the train is:1.4774174690246582\n",
      "0.942\n",
      "the 32500 setps AND the loss on the train is:1.497571587562561\n",
      "0.9426\n",
      "the 33000 setps AND the loss on the train is:1.5246732234954834\n",
      "0.9392\n",
      "the 33500 setps AND the loss on the train is:1.599435567855835\n",
      "0.9431\n",
      "the 34000 setps AND the loss on the train is:1.614511489868164\n",
      "0.9406\n",
      "the 34500 setps AND the loss on the train is:1.4616907835006714\n",
      "0.9404\n",
      "the 35000 setps AND the loss on the train is:1.5548431873321533\n",
      "0.9373\n",
      "the 35500 setps AND the loss on the train is:1.5369367599487305\n",
      "0.9396\n",
      "the 36000 setps AND the loss on the train is:1.60460364818573\n",
      "0.9401\n",
      "the 36500 setps AND the loss on the train is:1.4827194213867188\n",
      "0.9364\n",
      "the 37000 setps AND the loss on the train is:1.5967063903808594\n",
      "0.9395\n",
      "the 37500 setps AND the loss on the train is:1.5623270273208618\n",
      "0.941\n",
      "the 38000 setps AND the loss on the train is:1.5213816165924072\n",
      "0.9384\n",
      "the 38500 setps AND the loss on the train is:1.4722646474838257\n",
      "0.9418\n",
      "the 39000 setps AND the loss on the train is:1.5414831638336182\n",
      "0.9414\n",
      "the 39500 setps AND the loss on the train is:1.5286548137664795\n",
      "0.9421\n",
      "the 40000 setps AND the loss on the train is:1.5425225496292114\n",
      "0.9403\n",
      "the 40500 setps AND the loss on the train is:1.6292624473571777\n",
      "0.9349\n",
      "the 41000 setps AND the loss on the train is:1.5720494985580444\n",
      "0.9395\n",
      "the 41500 setps AND the loss on the train is:1.5647541284561157\n",
      "0.9419\n",
      "the 42000 setps AND the loss on the train is:1.526475429534912\n",
      "0.9401\n",
      "the 42500 setps AND the loss on the train is:1.6331753730773926\n",
      "0.9393\n",
      "the 43000 setps AND the loss on the train is:1.4977736473083496\n",
      "0.9441\n",
      "the 43500 setps AND the loss on the train is:1.5387868881225586\n",
      "0.9415\n",
      "the 44000 setps AND the loss on the train is:1.5533584356307983\n",
      "0.942\n",
      "the 44500 setps AND the loss on the train is:1.6305636167526245\n",
      "0.9417\n",
      "the 45000 setps AND the loss on the train is:1.5021438598632812\n",
      "0.9368\n",
      "the 45500 setps AND the loss on the train is:1.5410182476043701\n",
      "0.9396\n",
      "the 46000 setps AND the loss on the train is:1.5906438827514648\n",
      "0.942\n",
      "the 46500 setps AND the loss on the train is:1.4941507577896118\n",
      "0.94\n",
      "the 47000 setps AND the loss on the train is:1.5160133838653564\n",
      "0.9432\n",
      "the 47500 setps AND the loss on the train is:1.5118345022201538\n",
      "0.9404\n",
      "the 48000 setps AND the loss on the train is:1.5029925107955933\n",
      "0.9406\n",
      "the 48500 setps AND the loss on the train is:1.5031874179840088\n",
      "0.9423\n",
      "the 49000 setps AND the loss on the train is:1.563288927078247\n",
      "0.9424\n",
      "the 49500 setps AND the loss on the train is:1.6250146627426147\n",
      "0.9391\n",
      "the 50000 setps AND the loss on the train is:1.4978784322738647\n",
      "0.9404\n",
      "the 50500 setps AND the loss on the train is:1.6223723888397217\n",
      "0.9418\n",
      "the 51000 setps AND the loss on the train is:1.5336456298828125\n",
      "0.9399\n",
      "the 51500 setps AND the loss on the train is:1.5052129030227661\n",
      "0.9396\n",
      "the 52000 setps AND the loss on the train is:1.568107008934021\n",
      "0.9417\n",
      "the 52500 setps AND the loss on the train is:1.5504103899002075\n",
      "0.941\n",
      "the 53000 setps AND the loss on the train is:1.4874024391174316\n",
      "0.9411\n",
      "the 53500 setps AND the loss on the train is:1.6029976606369019\n",
      "0.9417\n",
      "the 54000 setps AND the loss on the train is:1.4628071784973145\n",
      "0.9413\n",
      "the 54500 setps AND the loss on the train is:1.5537962913513184\n",
      "0.9359\n",
      "the 55000 setps AND the loss on the train is:1.6258052587509155\n",
      "0.9398\n",
      "the 55500 setps AND the loss on the train is:1.51291823387146\n",
      "0.9418\n",
      "the 56000 setps AND the loss on the train is:1.5069921016693115\n",
      "0.9411\n",
      "the 56500 setps AND the loss on the train is:1.5295392274856567\n",
      "0.9401\n",
      "the 57000 setps AND the loss on the train is:1.5678175687789917\n",
      "0.9419\n",
      "the 57500 setps AND the loss on the train is:1.545436143875122\n",
      "0.9417\n",
      "the 58000 setps AND the loss on the train is:1.521162748336792\n",
      "0.9409\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 58500 setps AND the loss on the train is:1.4599207639694214\n",
      "0.94\n",
      "the 59000 setps AND the loss on the train is:1.5757720470428467\n",
      "0.9387\n",
      "the 59500 setps AND the loss on the train is:1.5124770402908325\n",
      "0.9398\n",
      "the 60000 setps AND the loss on the train is:1.6025075912475586\n",
      "0.9401\n",
      "the 60500 setps AND the loss on the train is:1.581626296043396\n",
      "0.9428\n",
      "the 61000 setps AND the loss on the train is:1.5199496746063232\n",
      "0.9421\n",
      "the 61500 setps AND the loss on the train is:1.4905143976211548\n",
      "0.9379\n",
      "the 62000 setps AND the loss on the train is:1.571141242980957\n",
      "0.9408\n",
      "the 62500 setps AND the loss on the train is:1.6059337854385376\n",
      "0.9418\n",
      "the 63000 setps AND the loss on the train is:1.5301482677459717\n",
      "0.9392\n",
      "the 63500 setps AND the loss on the train is:1.5673387050628662\n",
      "0.9401\n",
      "the 64000 setps AND the loss on the train is:1.5110161304473877\n",
      "0.9407\n",
      "the 64500 setps AND the loss on the train is:1.5553339719772339\n",
      "0.9414\n",
      "the 65000 setps AND the loss on the train is:1.5847994089126587\n",
      "0.9399\n",
      "the 65500 setps AND the loss on the train is:1.6216679811477661\n",
      "0.9395\n",
      "the 66000 setps AND the loss on the train is:1.5806548595428467\n",
      "0.9397\n",
      "the 66500 setps AND the loss on the train is:1.5155093669891357\n",
      "0.9414\n",
      "the 67000 setps AND the loss on the train is:1.5896928310394287\n",
      "0.939\n",
      "the 67500 setps AND the loss on the train is:1.5214948654174805\n",
      "0.9367\n",
      "the 68000 setps AND the loss on the train is:1.5265048742294312\n",
      "0.942\n",
      "the 68500 setps AND the loss on the train is:1.4770078659057617\n",
      "0.9411\n",
      "the 69000 setps AND the loss on the train is:1.6175771951675415\n",
      "0.9415\n",
      "the 69500 setps AND the loss on the train is:1.6959729194641113\n",
      "0.9392\n",
      "the 70000 setps AND the loss on the train is:1.5193679332733154\n",
      "0.9424\n",
      "the 70500 setps AND the loss on the train is:1.5433765649795532\n",
      "0.9389\n",
      "the 71000 setps AND the loss on the train is:1.6265251636505127\n",
      "0.9406\n",
      "the 71500 setps AND the loss on the train is:1.5924818515777588\n",
      "0.939\n",
      "the 72000 setps AND the loss on the train is:1.4796676635742188\n",
      "0.9419\n",
      "the 72500 setps AND the loss on the train is:1.5134732723236084\n",
      "0.9403\n",
      "the 73000 setps AND the loss on the train is:1.4822237491607666\n",
      "0.9392\n",
      "the 73500 setps AND the loss on the train is:1.487858772277832\n",
      "0.9415\n",
      "the 74000 setps AND the loss on the train is:1.5991140604019165\n",
      "0.9434\n",
      "the 74500 setps AND the loss on the train is:1.5504616498947144\n",
      "0.9397\n",
      "the 75000 setps AND the loss on the train is:1.5525062084197998\n",
      "0.9421\n",
      "the 75500 setps AND the loss on the train is:1.5258374214172363\n",
      "0.9412\n",
      "the 76000 setps AND the loss on the train is:1.6025625467300415\n",
      "0.9409\n",
      "the 76500 setps AND the loss on the train is:1.5321487188339233\n",
      "0.9389\n",
      "the 77000 setps AND the loss on the train is:1.6225937604904175\n",
      "0.9366\n",
      "the 77500 setps AND the loss on the train is:1.6248795986175537\n",
      "0.9404\n",
      "the 78000 setps AND the loss on the train is:1.6226909160614014\n",
      "0.9401\n",
      "the 78500 setps AND the loss on the train is:1.5843536853790283\n",
      "0.9405\n",
      "the 79000 setps AND the loss on the train is:1.5931742191314697\n",
      "0.9402\n",
      "the 79500 setps AND the loss on the train is:1.56688392162323\n",
      "0.9375\n",
      "the 80000 setps AND the loss on the train is:1.527060627937317\n",
      "0.9386\n",
      "the 80500 setps AND the loss on the train is:1.6071354150772095\n",
      "0.9407\n",
      "the 81000 setps AND the loss on the train is:1.5861420631408691\n",
      "0.9391\n",
      "the 81500 setps AND the loss on the train is:1.5051897764205933\n",
      "0.9402\n",
      "the 82000 setps AND the loss on the train is:1.5397430658340454\n",
      "0.9411\n",
      "the 82500 setps AND the loss on the train is:1.6168169975280762\n",
      "0.9433\n",
      "the 83000 setps AND the loss on the train is:1.5761301517486572\n",
      "0.9412\n",
      "the 83500 setps AND the loss on the train is:1.5447486639022827\n",
      "0.9384\n",
      "the 84000 setps AND the loss on the train is:1.5997658967971802\n",
      "0.9388\n",
      "the 84500 setps AND the loss on the train is:1.5847517251968384\n",
      "0.9416\n",
      "the 85000 setps AND the loss on the train is:1.5986104011535645\n",
      "0.9398\n",
      "the 85500 setps AND the loss on the train is:1.534865379333496\n",
      "0.9419\n",
      "the 86000 setps AND the loss on the train is:1.6097965240478516\n",
      "0.9394\n",
      "the 86500 setps AND the loss on the train is:1.5519739389419556\n",
      "0.9399\n",
      "the 87000 setps AND the loss on the train is:1.5003854036331177\n",
      "0.9364\n",
      "the 87500 setps AND the loss on the train is:1.5062557458877563\n",
      "0.9423\n",
      "the 88000 setps AND the loss on the train is:1.5099048614501953\n",
      "0.9416\n",
      "the 88500 setps AND the loss on the train is:1.5793099403381348\n",
      "0.9399\n",
      "the 89000 setps AND the loss on the train is:1.520376443862915\n",
      "0.9403\n",
      "the 89500 setps AND the loss on the train is:1.5507397651672363\n",
      "0.9432\n",
      "the 90000 setps AND the loss on the train is:1.5439256429672241\n",
      "0.9393\n",
      "the 90500 setps AND the loss on the train is:1.5001592636108398\n",
      "0.9398\n",
      "the 91000 setps AND the loss on the train is:1.5034348964691162\n",
      "0.9398\n",
      "the 91500 setps AND the loss on the train is:1.5031174421310425\n",
      "0.9319\n",
      "the 92000 setps AND the loss on the train is:1.5633976459503174\n",
      "0.9415\n",
      "the 92500 setps AND the loss on the train is:1.5442876815795898\n",
      "0.9403\n",
      "the 93000 setps AND the loss on the train is:1.5729197263717651\n",
      "0.9381\n",
      "the 93500 setps AND the loss on the train is:1.6286475658416748\n",
      "0.9415\n",
      "the 94000 setps AND the loss on the train is:1.5278620719909668\n",
      "0.9384\n",
      "the 94500 setps AND the loss on the train is:1.5681548118591309\n",
      "0.9387\n",
      "the 95000 setps AND the loss on the train is:1.4721133708953857\n",
      "0.9415\n",
      "the 95500 setps AND the loss on the train is:1.5701568126678467\n",
      "0.9397\n",
      "the 96000 setps AND the loss on the train is:1.5660148859024048\n",
      "0.9419\n",
      "the 96500 setps AND the loss on the train is:1.550277829170227\n",
      "0.9392\n",
      "the 97000 setps AND the loss on the train is:1.595411777496338\n",
      "0.9391\n",
      "the 97500 setps AND the loss on the train is:1.5932906866073608\n",
      "0.9421\n",
      "the 98000 setps AND the loss on the train is:1.6098272800445557\n",
      "0.9407\n",
      "the 98500 setps AND the loss on the train is:1.5748445987701416\n",
      "0.9386\n",
      "the 99000 setps AND the loss on the train is:1.5520368814468384\n",
      "0.9411\n",
      "the 99500 setps AND the loss on the train is:1.4851036071777344\n",
      "0.941\n",
      "the 100000 setps AND the loss on the train is:1.4740945100784302\n",
      "0.9372\n",
      "the 100500 setps AND the loss on the train is:1.5180892944335938\n",
      "0.9401\n",
      "the 101000 setps AND the loss on the train is:1.5222662687301636\n",
      "0.9411\n",
      "the 101500 setps AND the loss on the train is:1.543478012084961\n",
      "0.9388\n",
      "the 102000 setps AND the loss on the train is:1.5636032819747925\n",
      "0.9402\n",
      "the 102500 setps AND the loss on the train is:1.5722665786743164\n",
      "0.9394\n",
      "the 103000 setps AND the loss on the train is:1.5373835563659668\n",
      "0.9384\n",
      "the 103500 setps AND the loss on the train is:1.4862293004989624\n",
      "0.9402\n",
      "the 104000 setps AND the loss on the train is:1.5872873067855835\n",
      "0.9397\n",
      "the 104500 setps AND the loss on the train is:1.5497699975967407\n",
      "0.9394\n",
      "the 105000 setps AND the loss on the train is:1.516291856765747\n",
      "0.9408\n",
      "the 105500 setps AND the loss on the train is:1.538425087928772\n",
      "0.9383\n",
      "the 106000 setps AND the loss on the train is:1.5640493631362915\n",
      "0.9389\n",
      "the 106500 setps AND the loss on the train is:1.5297266244888306\n",
      "0.9415\n",
      "the 107000 setps AND the loss on the train is:1.5498322248458862\n",
      "0.9413\n",
      "the 107500 setps AND the loss on the train is:1.4581000804901123\n",
      "0.9383\n",
      "the 108000 setps AND the loss on the train is:1.648766040802002\n",
      "0.9407\n",
      "the 108500 setps AND the loss on the train is:1.5662471055984497\n",
      "0.9408\n",
      "the 109000 setps AND the loss on the train is:1.5739537477493286\n",
      "0.9421\n",
      "the 109500 setps AND the loss on the train is:1.538984775543213\n",
      "0.9404\n",
      "the 110000 setps AND the loss on the train is:1.5835455656051636\n",
      "0.9407\n",
      "the 110500 setps AND the loss on the train is:1.5245041847229004\n",
      "0.9406\n",
      "the 111000 setps AND the loss on the train is:1.504734754562378\n",
      "0.9423\n",
      "the 111500 setps AND the loss on the train is:1.5825530290603638\n",
      "0.9374\n",
      "the 112000 setps AND the loss on the train is:1.6058239936828613\n",
      "0.9404\n",
      "the 112500 setps AND the loss on the train is:1.546997308731079\n",
      "0.9417\n",
      "the 113000 setps AND the loss on the train is:1.5287935733795166\n",
      "0.9406\n",
      "the 113500 setps AND the loss on the train is:1.6201586723327637\n",
      "0.9395\n",
      "the 114000 setps AND the loss on the train is:1.5610700845718384\n",
      "0.9383\n",
      "the 114500 setps AND the loss on the train is:1.491014838218689\n",
      "0.9392\n",
      "the 115000 setps AND the loss on the train is:1.5271097421646118\n",
      "0.9413\n",
      "the 115500 setps AND the loss on the train is:1.5540637969970703\n",
      "0.9405\n",
      "the 116000 setps AND the loss on the train is:1.5167196989059448\n",
      "0.9414\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 116500 setps AND the loss on the train is:1.5971927642822266\n",
      "0.9364\n",
      "the 117000 setps AND the loss on the train is:1.6193268299102783\n",
      "0.9387\n",
      "the 117500 setps AND the loss on the train is:1.5412559509277344\n",
      "0.9407\n",
      "the 118000 setps AND the loss on the train is:1.494545578956604\n",
      "0.9402\n",
      "the 118500 setps AND the loss on the train is:1.613671898841858\n",
      "0.9407\n",
      "the 119000 setps AND the loss on the train is:1.4797286987304688\n",
      "0.9399\n",
      "the 119500 setps AND the loss on the train is:1.5591199398040771\n",
      "0.9385\n",
      "the 120000 setps AND the loss on the train is:1.6199740171432495\n",
      "0.9408\n",
      "the 120500 setps AND the loss on the train is:1.4822086095809937\n",
      "0.9371\n",
      "the 121000 setps AND the loss on the train is:1.5208053588867188\n",
      "0.9398\n",
      "the 121500 setps AND the loss on the train is:1.578292727470398\n",
      "0.9363\n",
      "the 122000 setps AND the loss on the train is:1.6296792030334473\n",
      "0.9405\n",
      "the 122500 setps AND the loss on the train is:1.5784924030303955\n",
      "0.9395\n",
      "the 123000 setps AND the loss on the train is:1.5308538675308228\n",
      "0.9402\n",
      "the 123500 setps AND the loss on the train is:1.4947335720062256\n",
      "0.9381\n",
      "the 124000 setps AND the loss on the train is:1.5662834644317627\n",
      "0.9406\n",
      "the 124500 setps AND the loss on the train is:1.4971530437469482\n",
      "0.9349\n",
      "the 125000 setps AND the loss on the train is:1.5349292755126953\n",
      "0.9406\n",
      "the 125500 setps AND the loss on the train is:1.560943841934204\n",
      "0.9373\n",
      "the 126000 setps AND the loss on the train is:1.5167007446289062\n",
      "0.9402\n",
      "the 126500 setps AND the loss on the train is:1.5453346967697144\n",
      "0.941\n",
      "the 127000 setps AND the loss on the train is:1.5848207473754883\n",
      "0.9399\n",
      "the 127500 setps AND the loss on the train is:1.5650262832641602\n",
      "0.9407\n",
      "the 128000 setps AND the loss on the train is:1.5201395750045776\n",
      "0.938\n",
      "the 128500 setps AND the loss on the train is:1.547151803970337\n",
      "0.9388\n",
      "the 129000 setps AND the loss on the train is:1.4723886251449585\n",
      "0.9413\n",
      "the 129500 setps AND the loss on the train is:1.5299479961395264\n",
      "0.9392\n",
      "the 130000 setps AND the loss on the train is:1.5250170230865479\n",
      "0.9393\n",
      "the 130500 setps AND the loss on the train is:1.528587818145752\n",
      "0.9411\n",
      "the 131000 setps AND the loss on the train is:1.5459346771240234\n",
      "0.9367\n",
      "the 131500 setps AND the loss on the train is:1.5927882194519043\n",
      "0.9389\n",
      "the 132000 setps AND the loss on the train is:1.5983580350875854\n",
      "0.9419\n",
      "the 132500 setps AND the loss on the train is:1.5904576778411865\n",
      "0.9406\n",
      "the 133000 setps AND the loss on the train is:1.606966257095337\n",
      "0.9395\n",
      "the 133500 setps AND the loss on the train is:1.5937479734420776\n",
      "0.94\n",
      "the 134000 setps AND the loss on the train is:1.5449784994125366\n",
      "0.9401\n",
      "the 134500 setps AND the loss on the train is:1.598585844039917\n",
      "0.94\n",
      "the 135000 setps AND the loss on the train is:1.5692987442016602\n",
      "0.9411\n",
      "the 135500 setps AND the loss on the train is:1.6065360307693481\n",
      "0.939\n",
      "the 136000 setps AND the loss on the train is:1.5641673803329468\n",
      "0.9386\n",
      "the 136500 setps AND the loss on the train is:1.5635753870010376\n",
      "0.942\n",
      "the 137000 setps AND the loss on the train is:1.7463184595108032\n",
      "0.9412\n",
      "the 137500 setps AND the loss on the train is:1.5076191425323486\n",
      "0.9401\n",
      "the 138000 setps AND the loss on the train is:1.5836498737335205\n",
      "0.9407\n",
      "the 138500 setps AND the loss on the train is:1.549379825592041\n",
      "0.9404\n",
      "the 139000 setps AND the loss on the train is:1.607919454574585\n",
      "0.9405\n",
      "the 139500 setps AND the loss on the train is:1.5191022157669067\n",
      "0.942\n",
      "the 140000 setps AND the loss on the train is:1.4826247692108154\n",
      "0.9399\n",
      "the 140500 setps AND the loss on the train is:1.5041584968566895\n",
      "0.9416\n",
      "the 141000 setps AND the loss on the train is:1.4523038864135742\n",
      "0.939\n",
      "the 141500 setps AND the loss on the train is:1.5060582160949707\n",
      "0.9379\n",
      "the 142000 setps AND the loss on the train is:1.4831215143203735\n",
      "0.939\n",
      "the 142500 setps AND the loss on the train is:1.5906943082809448\n",
      "0.9431\n",
      "the 143000 setps AND the loss on the train is:1.53655207157135\n",
      "0.9367\n",
      "the 143500 setps AND the loss on the train is:1.4710379838943481\n",
      "0.9407\n",
      "the 144000 setps AND the loss on the train is:1.5323129892349243\n",
      "0.9388\n",
      "the 144500 setps AND the loss on the train is:1.4931774139404297\n",
      "0.9392\n",
      "the 145000 setps AND the loss on the train is:1.530285358428955\n",
      "0.9396\n",
      "the 145500 setps AND the loss on the train is:1.4762338399887085\n",
      "0.9373\n",
      "the 146000 setps AND the loss on the train is:1.5438923835754395\n",
      "0.9376\n",
      "the 146500 setps AND the loss on the train is:1.4915038347244263\n",
      "0.938\n",
      "the 147000 setps AND the loss on the train is:1.5187851190567017\n",
      "0.939\n",
      "the 147500 setps AND the loss on the train is:1.5599137544631958\n",
      "0.9427\n",
      "the 148000 setps AND the loss on the train is:1.493431806564331\n",
      "0.9386\n",
      "the 148500 setps AND the loss on the train is:1.5435956716537476\n",
      "0.9393\n",
      "the 149000 setps AND the loss on the train is:1.4745620489120483\n",
      "0.9416\n",
      "the 149500 setps AND the loss on the train is:1.5223921537399292\n",
      "0.9395\n",
      "the 150000 setps AND the loss on the train is:1.5713531970977783\n",
      "0.9415\n",
      "the 150500 setps AND the loss on the train is:1.4848089218139648\n",
      "0.9409\n",
      "the 151000 setps AND the loss on the train is:1.5855515003204346\n",
      "0.939\n",
      "the 151500 setps AND the loss on the train is:1.5335032939910889\n",
      "0.9412\n",
      "the 152000 setps AND the loss on the train is:1.5029959678649902\n",
      "0.94\n",
      "the 152500 setps AND the loss on the train is:1.5436052083969116\n",
      "0.9415\n",
      "the 153000 setps AND the loss on the train is:1.4562472105026245\n",
      "0.9405\n",
      "the 153500 setps AND the loss on the train is:1.5476540327072144\n",
      "0.9378\n",
      "the 154000 setps AND the loss on the train is:1.5541644096374512\n",
      "0.9386\n",
      "the 154500 setps AND the loss on the train is:1.5564879179000854\n",
      "0.9411\n",
      "the 155000 setps AND the loss on the train is:1.5209294557571411\n",
      "0.9408\n",
      "the 155500 setps AND the loss on the train is:1.5926499366760254\n",
      "0.9372\n",
      "the 156000 setps AND the loss on the train is:1.6126075983047485\n",
      "0.9426\n",
      "the 156500 setps AND the loss on the train is:1.601147174835205\n",
      "0.9355\n",
      "the 157000 setps AND the loss on the train is:1.5147781372070312\n",
      "0.9413\n",
      "the 157500 setps AND the loss on the train is:1.6043603420257568\n",
      "0.9401\n",
      "the 158000 setps AND the loss on the train is:1.4753929376602173\n",
      "0.9409\n",
      "the 158500 setps AND the loss on the train is:1.620395302772522\n",
      "0.9406\n",
      "the 159000 setps AND the loss on the train is:1.5405724048614502\n",
      "0.939\n",
      "the 159500 setps AND the loss on the train is:1.6283605098724365\n",
      "0.9401\n",
      "the 160000 setps AND the loss on the train is:1.5734695196151733\n",
      "0.9359\n",
      "the 160500 setps AND the loss on the train is:1.5306299924850464\n",
      "0.9426\n",
      "the 161000 setps AND the loss on the train is:1.5065670013427734\n",
      "0.9405\n",
      "the 161500 setps AND the loss on the train is:1.486769437789917\n",
      "0.9393\n",
      "the 162000 setps AND the loss on the train is:1.5320799350738525\n",
      "0.9364\n",
      "the 162500 setps AND the loss on the train is:1.6020416021347046\n",
      "0.9428\n",
      "the 163000 setps AND the loss on the train is:1.493503212928772\n",
      "0.9402\n",
      "the 163500 setps AND the loss on the train is:1.475475788116455\n",
      "0.9382\n",
      "the 164000 setps AND the loss on the train is:1.5345797538757324\n",
      "0.9402\n",
      "the 164500 setps AND the loss on the train is:1.5968838930130005\n",
      "0.9413\n",
      "the 165000 setps AND the loss on the train is:1.6024489402770996\n",
      "0.9408\n",
      "the 165500 setps AND the loss on the train is:1.592060923576355\n",
      "0.9403\n",
      "the 166000 setps AND the loss on the train is:1.5263643264770508\n",
      "0.9411\n",
      "the 166500 setps AND the loss on the train is:1.6236332654953003\n",
      "0.9399\n",
      "the 167000 setps AND the loss on the train is:1.5926990509033203\n",
      "0.9405\n",
      "the 167500 setps AND the loss on the train is:1.4874508380889893\n",
      "0.9381\n",
      "the 168000 setps AND the loss on the train is:1.5691049098968506\n",
      "0.9382\n",
      "the 168500 setps AND the loss on the train is:1.5692917108535767\n",
      "0.9383\n",
      "the 169000 setps AND the loss on the train is:1.5933083295822144\n",
      "0.9394\n",
      "the 169500 setps AND the loss on the train is:1.4929769039154053\n",
      "0.9399\n",
      "the 170000 setps AND the loss on the train is:1.679956316947937\n",
      "0.9383\n",
      "the 170500 setps AND the loss on the train is:1.43826162815094\n",
      "0.9404\n",
      "the 171000 setps AND the loss on the train is:1.5070942640304565\n",
      "0.942\n",
      "the 171500 setps AND the loss on the train is:1.6419163942337036\n",
      "0.94\n",
      "the 172000 setps AND the loss on the train is:1.551554560661316\n",
      "0.9405\n",
      "the 172500 setps AND the loss on the train is:1.5768048763275146\n",
      "0.9397\n",
      "the 173000 setps AND the loss on the train is:1.5155000686645508\n",
      "0.9402\n",
      "the 173500 setps AND the loss on the train is:1.5347340106964111\n",
      "0.9417\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 174000 setps AND the loss on the train is:1.5300955772399902\n",
      "0.9395\n",
      "the 174500 setps AND the loss on the train is:1.6556265354156494\n",
      "0.9413\n",
      "the 175000 setps AND the loss on the train is:1.5171412229537964\n",
      "0.9427\n",
      "the 175500 setps AND the loss on the train is:1.585706353187561\n",
      "0.9417\n",
      "the 176000 setps AND the loss on the train is:1.586165189743042\n",
      "0.941\n",
      "the 176500 setps AND the loss on the train is:1.5714796781539917\n",
      "0.9406\n",
      "the 177000 setps AND the loss on the train is:1.521714210510254\n",
      "0.9404\n",
      "the 177500 setps AND the loss on the train is:1.5664422512054443\n",
      "0.9395\n",
      "the 178000 setps AND the loss on the train is:1.6205683946609497\n",
      "0.9402\n",
      "the 178500 setps AND the loss on the train is:1.659456491470337\n",
      "0.9416\n",
      "the 179000 setps AND the loss on the train is:1.515039324760437\n",
      "0.9416\n",
      "the 179500 setps AND the loss on the train is:1.5054450035095215\n",
      "0.9396\n",
      "the 180000 setps AND the loss on the train is:1.535892367362976\n",
      "0.9394\n",
      "the 180500 setps AND the loss on the train is:1.564552664756775\n",
      "0.9367\n",
      "the 181000 setps AND the loss on the train is:1.573230266571045\n",
      "0.9402\n",
      "the 181500 setps AND the loss on the train is:1.5341720581054688\n",
      "0.9428\n",
      "the 182000 setps AND the loss on the train is:1.59266197681427\n",
      "0.9422\n",
      "the 182500 setps AND the loss on the train is:1.5241146087646484\n",
      "0.9403\n",
      "the 183000 setps AND the loss on the train is:1.5317885875701904\n",
      "0.9409\n",
      "the 183500 setps AND the loss on the train is:1.5735567808151245\n",
      "0.9399\n",
      "the 184000 setps AND the loss on the train is:1.5671168565750122\n",
      "0.9412\n",
      "the 184500 setps AND the loss on the train is:1.6522983312606812\n",
      "0.9403\n",
      "the 185000 setps AND the loss on the train is:1.5820603370666504\n",
      "0.942\n",
      "the 185500 setps AND the loss on the train is:1.554355502128601\n",
      "0.9426\n",
      "the 186000 setps AND the loss on the train is:1.6294834613800049\n",
      "0.9353\n",
      "the 186500 setps AND the loss on the train is:1.6066844463348389\n",
      "0.9406\n",
      "the 187000 setps AND the loss on the train is:1.5310317277908325\n",
      "0.9425\n",
      "the 187500 setps AND the loss on the train is:1.5800009965896606\n",
      "0.9425\n",
      "the 188000 setps AND the loss on the train is:1.6017757654190063\n",
      "0.9377\n",
      "the 188500 setps AND the loss on the train is:1.6016074419021606\n",
      "0.942\n",
      "the 189000 setps AND the loss on the train is:1.5836461782455444\n",
      "0.9417\n",
      "the 189500 setps AND the loss on the train is:1.507179856300354\n",
      "0.9402\n",
      "the 190000 setps AND the loss on the train is:1.5220558643341064\n",
      "0.9399\n",
      "the 190500 setps AND the loss on the train is:1.5853418111801147\n",
      "0.9394\n",
      "the 191000 setps AND the loss on the train is:1.5185036659240723\n",
      "0.9376\n",
      "the 191500 setps AND the loss on the train is:1.5940475463867188\n",
      "0.9425\n",
      "the 192000 setps AND the loss on the train is:1.5312262773513794\n",
      "0.9428\n",
      "the 192500 setps AND the loss on the train is:1.5072509050369263\n",
      "0.9408\n",
      "the 193000 setps AND the loss on the train is:1.5360153913497925\n",
      "0.9419\n",
      "the 193500 setps AND the loss on the train is:1.4879266023635864\n",
      "0.9421\n",
      "the 194000 setps AND the loss on the train is:1.5908312797546387\n",
      "0.9391\n",
      "the 194500 setps AND the loss on the train is:1.5114161968231201\n",
      "0.9377\n",
      "the 195000 setps AND the loss on the train is:1.6139450073242188\n",
      "0.9418\n",
      "the 195500 setps AND the loss on the train is:1.5855578184127808\n",
      "0.9414\n",
      "the 196000 setps AND the loss on the train is:1.552069902420044\n",
      "0.9407\n",
      "the 196500 setps AND the loss on the train is:1.5587900876998901\n",
      "0.9418\n",
      "the 197000 setps AND the loss on the train is:1.5832852125167847\n",
      "0.9397\n",
      "the 197500 setps AND the loss on the train is:1.5158973932266235\n",
      "0.9416\n",
      "the 198000 setps AND the loss on the train is:1.58732271194458\n",
      "0.9422\n",
      "the 198500 setps AND the loss on the train is:1.6154992580413818\n",
      "0.939\n",
      "the 199000 setps AND the loss on the train is:1.5117337703704834\n",
      "0.9405\n",
      "the 199500 setps AND the loss on the train is:1.5473694801330566\n",
      "0.9415\n",
      "the 200000 setps AND the loss on the train is:1.593193531036377\n",
      "0.9385\n",
      "the 200500 setps AND the loss on the train is:1.5790247917175293\n",
      "0.9396\n",
      "the 201000 setps AND the loss on the train is:1.5783443450927734\n",
      "0.9411\n",
      "the 201500 setps AND the loss on the train is:1.6066977977752686\n",
      "0.94\n",
      "the 202000 setps AND the loss on the train is:1.5632644891738892\n",
      "0.9388\n",
      "the 202500 setps AND the loss on the train is:1.5066672563552856\n",
      "0.9413\n",
      "the 203000 setps AND the loss on the train is:1.6124290227890015\n",
      "0.9377\n",
      "the 203500 setps AND the loss on the train is:1.6051499843597412\n",
      "0.9418\n",
      "the 204000 setps AND the loss on the train is:1.5824172496795654\n",
      "0.9421\n",
      "the 204500 setps AND the loss on the train is:1.5714064836502075\n",
      "0.9423\n",
      "the 205000 setps AND the loss on the train is:1.5551835298538208\n",
      "0.9407\n",
      "the 205500 setps AND the loss on the train is:1.5672979354858398\n",
      "0.9396\n",
      "the 206000 setps AND the loss on the train is:1.4847251176834106\n",
      "0.9418\n",
      "the 206500 setps AND the loss on the train is:1.480162501335144\n",
      "0.9388\n",
      "the 207000 setps AND the loss on the train is:1.5389877557754517\n",
      "0.941\n",
      "the 207500 setps AND the loss on the train is:1.4995570182800293\n",
      "0.9416\n",
      "the 208000 setps AND the loss on the train is:1.523044228553772\n",
      "0.94\n",
      "the 208500 setps AND the loss on the train is:1.4966790676116943\n",
      "0.9397\n",
      "the 209000 setps AND the loss on the train is:1.6190544366836548\n",
      "0.9378\n",
      "the 209500 setps AND the loss on the train is:1.5091760158538818\n",
      "0.94\n",
      "the 210000 setps AND the loss on the train is:1.5567355155944824\n",
      "0.9406\n",
      "the 210500 setps AND the loss on the train is:1.550621747970581\n",
      "0.94\n",
      "the 211000 setps AND the loss on the train is:1.5127252340316772\n",
      "0.9393\n",
      "the 211500 setps AND the loss on the train is:1.592463493347168\n",
      "0.943\n",
      "the 212000 setps AND the loss on the train is:1.5360300540924072\n",
      "0.9384\n",
      "the 212500 setps AND the loss on the train is:1.5555006265640259\n",
      "0.9409\n",
      "the 213000 setps AND the loss on the train is:1.5173988342285156\n",
      "0.9397\n",
      "the 213500 setps AND the loss on the train is:1.5598485469818115\n",
      "0.9394\n",
      "the 214000 setps AND the loss on the train is:1.5123757123947144\n",
      "0.9361\n",
      "the 214500 setps AND the loss on the train is:1.4852898120880127\n",
      "0.94\n",
      "the 215000 setps AND the loss on the train is:1.560481071472168\n",
      "0.9402\n",
      "the 215500 setps AND the loss on the train is:1.5225203037261963\n",
      "0.9392\n",
      "the 216000 setps AND the loss on the train is:1.5074163675308228\n",
      "0.9386\n",
      "the 216500 setps AND the loss on the train is:1.610824465751648\n",
      "0.9396\n",
      "the 217000 setps AND the loss on the train is:1.5178686380386353\n",
      "0.9395\n",
      "the 217500 setps AND the loss on the train is:1.5651582479476929\n",
      "0.9408\n",
      "the 218000 setps AND the loss on the train is:1.674499750137329\n",
      "0.9372\n",
      "the 218500 setps AND the loss on the train is:1.5895156860351562\n",
      "0.9399\n",
      "the 219000 setps AND the loss on the train is:1.6215691566467285\n",
      "0.9378\n",
      "the 219500 setps AND the loss on the train is:1.544699788093567\n",
      "0.9395\n",
      "the 220000 setps AND the loss on the train is:1.502042531967163\n",
      "0.9404\n",
      "the 220500 setps AND the loss on the train is:1.5218347311019897\n",
      "0.9416\n",
      "the 221000 setps AND the loss on the train is:1.6287634372711182\n",
      "0.9398\n",
      "the 221500 setps AND the loss on the train is:1.534432291984558\n",
      "0.939\n",
      "the 222000 setps AND the loss on the train is:1.508094072341919\n",
      "0.9398\n",
      "the 222500 setps AND the loss on the train is:1.5022764205932617\n",
      "0.9379\n",
      "the 223000 setps AND the loss on the train is:1.6580610275268555\n",
      "0.9432\n",
      "the 223500 setps AND the loss on the train is:1.5984387397766113\n",
      "0.9405\n",
      "the 224000 setps AND the loss on the train is:1.5567166805267334\n",
      "0.9406\n",
      "the 224500 setps AND the loss on the train is:1.5845518112182617\n",
      "0.9382\n",
      "the 225000 setps AND the loss on the train is:1.5409785509109497\n",
      "0.9431\n",
      "the 225500 setps AND the loss on the train is:1.6141878366470337\n",
      "0.9365\n",
      "the 226000 setps AND the loss on the train is:1.5147351026535034\n",
      "0.9412\n",
      "the 226500 setps AND the loss on the train is:1.5898815393447876\n",
      "0.9395\n",
      "the 227000 setps AND the loss on the train is:1.5433919429779053\n",
      "0.9414\n",
      "the 227500 setps AND the loss on the train is:1.5899581909179688\n",
      "0.9394\n",
      "the 228000 setps AND the loss on the train is:1.5709266662597656\n",
      "0.9387\n",
      "the 228500 setps AND the loss on the train is:1.5338102579116821\n",
      "0.9396\n",
      "the 229000 setps AND the loss on the train is:1.4991464614868164\n",
      "0.9381\n",
      "the 229500 setps AND the loss on the train is:1.6101328134536743\n",
      "0.9313\n",
      "the 230000 setps AND the loss on the train is:1.5055090188980103\n",
      "0.9388\n",
      "the 230500 setps AND the loss on the train is:1.56013822555542\n",
      "0.9402\n",
      "the 231000 setps AND the loss on the train is:1.5096049308776855\n",
      "0.9422\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 231500 setps AND the loss on the train is:1.5426664352416992\n",
      "0.9404\n",
      "the 232000 setps AND the loss on the train is:1.5878089666366577\n",
      "0.9425\n",
      "the 232500 setps AND the loss on the train is:1.6465773582458496\n",
      "0.939\n",
      "the 233000 setps AND the loss on the train is:1.6140199899673462\n",
      "0.9419\n",
      "the 233500 setps AND the loss on the train is:1.592218279838562\n",
      "0.9399\n",
      "the 234000 setps AND the loss on the train is:1.5370489358901978\n",
      "0.9397\n",
      "the 234500 setps AND the loss on the train is:1.5795332193374634\n",
      "0.9395\n",
      "the 235000 setps AND the loss on the train is:1.5593481063842773\n",
      "0.9368\n",
      "the 235500 setps AND the loss on the train is:1.5341780185699463\n",
      "0.9436\n",
      "the 236000 setps AND the loss on the train is:1.5665174722671509\n",
      "0.9421\n",
      "the 236500 setps AND the loss on the train is:1.5855658054351807\n",
      "0.9443\n",
      "the 237000 setps AND the loss on the train is:1.5344632863998413\n",
      "0.9397\n",
      "the 237500 setps AND the loss on the train is:1.5321648120880127\n",
      "0.9428\n",
      "the 238000 setps AND the loss on the train is:1.5035394430160522\n",
      "0.9414\n",
      "the 238500 setps AND the loss on the train is:1.5435117483139038\n",
      "0.9398\n",
      "the 239000 setps AND the loss on the train is:1.531571865081787\n",
      "0.9378\n",
      "the 239500 setps AND the loss on the train is:1.5494359731674194\n",
      "0.937\n",
      "the 240000 setps AND the loss on the train is:1.5070894956588745\n",
      "0.9379\n",
      "the 240500 setps AND the loss on the train is:1.562256097793579\n",
      "0.9402\n",
      "the 241000 setps AND the loss on the train is:1.5545144081115723\n",
      "0.9412\n",
      "the 241500 setps AND the loss on the train is:1.5572738647460938\n",
      "0.9393\n",
      "the 242000 setps AND the loss on the train is:1.5947538614273071\n",
      "0.9399\n",
      "the 242500 setps AND the loss on the train is:1.5998668670654297\n",
      "0.9417\n",
      "the 243000 setps AND the loss on the train is:1.5073105096817017\n",
      "0.9396\n",
      "the 243500 setps AND the loss on the train is:1.5158710479736328\n",
      "0.9397\n",
      "the 244000 setps AND the loss on the train is:1.640525221824646\n",
      "0.9395\n",
      "the 244500 setps AND the loss on the train is:1.6200745105743408\n",
      "0.9409\n",
      "the 245000 setps AND the loss on the train is:1.6587486267089844\n",
      "0.9408\n",
      "the 245500 setps AND the loss on the train is:1.5794774293899536\n",
      "0.9401\n",
      "the 246000 setps AND the loss on the train is:1.576290249824524\n",
      "0.9391\n",
      "the 246500 setps AND the loss on the train is:1.5459972620010376\n",
      "0.9395\n",
      "the 247000 setps AND the loss on the train is:1.5991625785827637\n",
      "0.9404\n",
      "the 247500 setps AND the loss on the train is:1.5409736633300781\n",
      "0.9389\n",
      "the 248000 setps AND the loss on the train is:1.5906198024749756\n",
      "0.9399\n",
      "the 248500 setps AND the loss on the train is:1.5206741094589233\n",
      "0.9422\n",
      "the 249000 setps AND the loss on the train is:1.5221152305603027\n",
      "0.9401\n",
      "the 249500 setps AND the loss on the train is:1.5781373977661133\n",
      "0.9385\n",
      "the 250000 setps AND the loss on the train is:1.5479670763015747\n",
      "0.9414\n",
      "the 250500 setps AND the loss on the train is:1.5611952543258667\n",
      "0.9407\n",
      "the 251000 setps AND the loss on the train is:1.5823309421539307\n",
      "0.9412\n",
      "the 251500 setps AND the loss on the train is:1.4902762174606323\n",
      "0.939\n",
      "the 252000 setps AND the loss on the train is:1.562026858329773\n",
      "0.9396\n",
      "the 252500 setps AND the loss on the train is:1.5180320739746094\n",
      "0.9407\n",
      "the 253000 setps AND the loss on the train is:1.5549335479736328\n",
      "0.9414\n",
      "the 253500 setps AND the loss on the train is:1.561360478401184\n",
      "0.9407\n",
      "the 254000 setps AND the loss on the train is:1.5335121154785156\n",
      "0.9411\n",
      "the 254500 setps AND the loss on the train is:1.503664255142212\n",
      "0.9393\n",
      "the 255000 setps AND the loss on the train is:1.5582153797149658\n",
      "0.9401\n",
      "the 255500 setps AND the loss on the train is:1.5016024112701416\n",
      "0.9412\n",
      "the 256000 setps AND the loss on the train is:1.4880611896514893\n",
      "0.9422\n",
      "the 256500 setps AND the loss on the train is:1.5326411724090576\n",
      "0.9424\n",
      "the 257000 setps AND the loss on the train is:1.5744999647140503\n",
      "0.9398\n",
      "the 257500 setps AND the loss on the train is:1.543428659439087\n",
      "0.9385\n",
      "the 258000 setps AND the loss on the train is:1.5256054401397705\n",
      "0.9406\n",
      "the 258500 setps AND the loss on the train is:1.5162240266799927\n",
      "0.9413\n",
      "the 259000 setps AND the loss on the train is:1.5695576667785645\n",
      "0.9372\n",
      "the 259500 setps AND the loss on the train is:1.569191813468933\n",
      "0.9409\n",
      "the 260000 setps AND the loss on the train is:1.5167579650878906\n",
      "0.9392\n",
      "the 260500 setps AND the loss on the train is:1.4483087062835693\n",
      "0.9413\n",
      "the 261000 setps AND the loss on the train is:1.5168536901474\n",
      "0.9396\n",
      "the 261500 setps AND the loss on the train is:1.5335842370986938\n",
      "0.941\n",
      "the 262000 setps AND the loss on the train is:1.511425495147705\n",
      "0.9398\n",
      "the 262500 setps AND the loss on the train is:1.5858230590820312\n",
      "0.9389\n",
      "the 263000 setps AND the loss on the train is:1.5686317682266235\n",
      "0.9399\n",
      "the 263500 setps AND the loss on the train is:1.5954949855804443\n",
      "0.9379\n",
      "the 264000 setps AND the loss on the train is:1.5580475330352783\n",
      "0.9421\n",
      "the 264500 setps AND the loss on the train is:1.5228389501571655\n",
      "0.9396\n",
      "the 265000 setps AND the loss on the train is:1.5304484367370605\n",
      "0.9396\n",
      "the 265500 setps AND the loss on the train is:1.4833836555480957\n",
      "0.9389\n",
      "the 266000 setps AND the loss on the train is:1.555059552192688\n",
      "0.9415\n",
      "the 266500 setps AND the loss on the train is:1.5301740169525146\n",
      "0.9359\n",
      "the 267000 setps AND the loss on the train is:1.579880952835083\n",
      "0.9394\n",
      "the 267500 setps AND the loss on the train is:1.5067634582519531\n",
      "0.9365\n",
      "the 268000 setps AND the loss on the train is:1.508175015449524\n",
      "0.9387\n",
      "the 268500 setps AND the loss on the train is:1.5692203044891357\n",
      "0.9411\n",
      "the 269000 setps AND the loss on the train is:1.5470714569091797\n",
      "0.9405\n",
      "the 269500 setps AND the loss on the train is:1.5581350326538086\n",
      "0.9396\n",
      "the 270000 setps AND the loss on the train is:1.5839643478393555\n",
      "0.9414\n",
      "the 270500 setps AND the loss on the train is:1.555314302444458\n",
      "0.9348\n",
      "the 271000 setps AND the loss on the train is:1.5503101348876953\n",
      "0.9405\n",
      "the 271500 setps AND the loss on the train is:1.6037366390228271\n",
      "0.9417\n",
      "the 272000 setps AND the loss on the train is:1.570162296295166\n",
      "0.9402\n",
      "the 272500 setps AND the loss on the train is:1.5151110887527466\n",
      "0.9389\n",
      "the 273000 setps AND the loss on the train is:1.5158636569976807\n",
      "0.9407\n",
      "the 273500 setps AND the loss on the train is:1.4628243446350098\n",
      "0.9407\n",
      "the 274000 setps AND the loss on the train is:1.6177337169647217\n",
      "0.9406\n",
      "the 274500 setps AND the loss on the train is:1.5640857219696045\n",
      "0.9404\n",
      "the 275000 setps AND the loss on the train is:1.5449721813201904\n",
      "0.9403\n",
      "the 275500 setps AND the loss on the train is:1.5979894399642944\n",
      "0.943\n",
      "the 276000 setps AND the loss on the train is:1.582154631614685\n",
      "0.9396\n",
      "the 276500 setps AND the loss on the train is:1.6150357723236084\n",
      "0.9395\n",
      "the 277000 setps AND the loss on the train is:1.4775391817092896\n",
      "0.9385\n",
      "the 277500 setps AND the loss on the train is:1.5220524072647095\n",
      "0.9401\n",
      "the 278000 setps AND the loss on the train is:1.4760730266571045\n",
      "0.9393\n",
      "the 278500 setps AND the loss on the train is:1.6025911569595337\n",
      "0.9411\n",
      "the 279000 setps AND the loss on the train is:1.4942522048950195\n",
      "0.9395\n",
      "the 279500 setps AND the loss on the train is:1.552116870880127\n",
      "0.9396\n",
      "the 280000 setps AND the loss on the train is:1.5964384078979492\n",
      "0.9421\n",
      "the 280500 setps AND the loss on the train is:1.5483890771865845\n",
      "0.9391\n",
      "the 281000 setps AND the loss on the train is:1.5280821323394775\n",
      "0.9411\n",
      "the 281500 setps AND the loss on the train is:1.4605484008789062\n",
      "0.9401\n",
      "the 282000 setps AND the loss on the train is:1.5399305820465088\n",
      "0.9403\n",
      "the 282500 setps AND the loss on the train is:1.5511869192123413\n",
      "0.9385\n",
      "the 283000 setps AND the loss on the train is:1.577804684638977\n",
      "0.9384\n",
      "the 283500 setps AND the loss on the train is:1.5345399379730225\n",
      "0.9418\n",
      "the 284000 setps AND the loss on the train is:1.5402886867523193\n",
      "0.9417\n",
      "the 284500 setps AND the loss on the train is:1.573603630065918\n",
      "0.9401\n",
      "the 285000 setps AND the loss on the train is:1.4944812059402466\n",
      "0.9379\n",
      "the 285500 setps AND the loss on the train is:1.562703013420105\n",
      "0.9391\n",
      "the 286000 setps AND the loss on the train is:1.4878146648406982\n",
      "0.9388\n",
      "the 286500 setps AND the loss on the train is:1.6514170169830322\n",
      "0.9411\n",
      "the 287000 setps AND the loss on the train is:1.5836044549942017\n",
      "0.9412\n",
      "the 287500 setps AND the loss on the train is:1.5284957885742188\n",
      "0.9386\n",
      "the 288000 setps AND the loss on the train is:1.4911221265792847\n",
      "0.9406\n",
      "the 288500 setps AND the loss on the train is:1.4764422178268433\n",
      "0.9402\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 289000 setps AND the loss on the train is:1.6152111291885376\n",
      "0.9407\n",
      "the 289500 setps AND the loss on the train is:1.6142456531524658\n",
      "0.9367\n",
      "the 290000 setps AND the loss on the train is:1.4976470470428467\n",
      "0.9396\n",
      "the 290500 setps AND the loss on the train is:1.5431259870529175\n",
      "0.9393\n",
      "the 291000 setps AND the loss on the train is:1.5932409763336182\n",
      "0.9417\n",
      "the 291500 setps AND the loss on the train is:1.638132095336914\n",
      "0.9395\n",
      "the 292000 setps AND the loss on the train is:1.5031867027282715\n",
      "0.9408\n",
      "the 292500 setps AND the loss on the train is:1.7100231647491455\n",
      "0.9382\n",
      "the 293000 setps AND the loss on the train is:1.4982950687408447\n",
      "0.9401\n",
      "the 293500 setps AND the loss on the train is:1.560481071472168\n",
      "0.9416\n",
      "the 294000 setps AND the loss on the train is:1.5322707891464233\n",
      "0.9402\n",
      "the 294500 setps AND the loss on the train is:1.5920716524124146\n",
      "0.9381\n",
      "the 295000 setps AND the loss on the train is:1.575765609741211\n",
      "0.9404\n",
      "the 295500 setps AND the loss on the train is:1.6442809104919434\n",
      "0.9389\n",
      "the 296000 setps AND the loss on the train is:1.5003788471221924\n",
      "0.9408\n",
      "the 296500 setps AND the loss on the train is:1.5360082387924194\n",
      "0.9405\n",
      "the 297000 setps AND the loss on the train is:1.5732835531234741\n",
      "0.9422\n",
      "the 297500 setps AND the loss on the train is:1.5542750358581543\n",
      "0.9395\n",
      "the 298000 setps AND the loss on the train is:1.6149972677230835\n",
      "0.9401\n",
      "the 298500 setps AND the loss on the train is:1.5407874584197998\n",
      "0.9432\n",
      "the 299000 setps AND the loss on the train is:1.5332826375961304\n",
      "0.9402\n",
      "the 299500 setps AND the loss on the train is:1.586835503578186\n",
      "0.9409\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,50],0.01)\n",
    "b1=tf.Variable(tf.random_normal([50]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.tanh(logits1)\n",
    "\n",
    "w2=get_weight([50,50],0.01)\n",
    "b2=tf.Variable(tf.random_normal([50]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.tanh(logits2)\n",
    "\n",
    "w3=get_weight([50,10],0.01)\n",
    "b3=tf.Variable(tf.random_normal([10]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "\n",
    "global_step=tf.Variable(0,trainable=False)\n",
    "learning_rate=tf.train.exponential_decay(0.1,global_step,500,0.99)\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits3))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits3,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(300000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4.4使用两个隐层，tanh激活，激活cells从50和50调整到512和256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:10030.822265625\n",
      "0.1379\n",
      "the 500 setps AND the loss on the train is:3629.122314453125\n",
      "0.8215\n",
      "the 1000 setps AND the loss on the train is:1300.9322509765625\n",
      "0.8867\n",
      "the 1500 setps AND the loss on the train is:471.5011291503906\n",
      "0.9254\n",
      "the 2000 setps AND the loss on the train is:168.61434936523438\n",
      "0.938\n",
      "the 2500 setps AND the loss on the train is:60.227996826171875\n",
      "0.939\n",
      "the 3000 setps AND the loss on the train is:22.302371978759766\n",
      "0.9355\n",
      "the 3500 setps AND the loss on the train is:8.92346477508545\n",
      "0.9238\n",
      "the 4000 setps AND the loss on the train is:4.199350833892822\n",
      "0.9245\n",
      "the 4500 setps AND the loss on the train is:2.599919557571411\n",
      "0.9233\n",
      "the 5000 setps AND the loss on the train is:1.9733096361160278\n",
      "0.9186\n",
      "the 5500 setps AND the loss on the train is:1.7694709300994873\n",
      "0.9191\n",
      "the 6000 setps AND the loss on the train is:1.560657262802124\n",
      "0.9016\n",
      "the 6500 setps AND the loss on the train is:1.5093743801116943\n",
      "0.9275\n",
      "the 7000 setps AND the loss on the train is:1.4968513250350952\n",
      "0.9293\n",
      "the 7500 setps AND the loss on the train is:1.4307823181152344\n",
      "0.9266\n",
      "the 8000 setps AND the loss on the train is:1.5632184743881226\n",
      "0.9272\n",
      "the 8500 setps AND the loss on the train is:1.5337245464324951\n",
      "0.9237\n",
      "the 9000 setps AND the loss on the train is:1.4577338695526123\n",
      "0.9248\n",
      "the 9500 setps AND the loss on the train is:1.552289605140686\n",
      "0.9304\n",
      "the 10000 setps AND the loss on the train is:1.5351622104644775\n",
      "0.9335\n",
      "the 10500 setps AND the loss on the train is:1.559418797492981\n",
      "0.9278\n",
      "the 11000 setps AND the loss on the train is:1.500855803489685\n",
      "0.9261\n",
      "the 11500 setps AND the loss on the train is:1.5145347118377686\n",
      "0.9261\n",
      "the 12000 setps AND the loss on the train is:1.5101042985916138\n",
      "0.9289\n",
      "the 12500 setps AND the loss on the train is:1.5116750001907349\n",
      "0.9232\n",
      "the 13000 setps AND the loss on the train is:1.5826690196990967\n",
      "0.9291\n",
      "the 13500 setps AND the loss on the train is:1.4984242916107178\n",
      "0.9286\n",
      "the 14000 setps AND the loss on the train is:1.54352605342865\n",
      "0.9271\n",
      "the 14500 setps AND the loss on the train is:1.589836835861206\n",
      "0.9294\n",
      "the 15000 setps AND the loss on the train is:1.534532904624939\n",
      "0.9301\n",
      "the 15500 setps AND the loss on the train is:1.6003867387771606\n",
      "0.926\n",
      "the 16000 setps AND the loss on the train is:1.4884179830551147\n",
      "0.9311\n",
      "the 16500 setps AND the loss on the train is:1.507217288017273\n",
      "0.9322\n",
      "the 17000 setps AND the loss on the train is:1.5934808254241943\n",
      "0.9245\n",
      "the 17500 setps AND the loss on the train is:1.5811715126037598\n",
      "0.9332\n",
      "the 18000 setps AND the loss on the train is:1.495888352394104\n",
      "0.9313\n",
      "the 18500 setps AND the loss on the train is:1.4657189846038818\n",
      "0.9293\n",
      "the 19000 setps AND the loss on the train is:1.668464183807373\n",
      "0.9294\n",
      "the 19500 setps AND the loss on the train is:1.52522611618042\n",
      "0.9304\n",
      "the 20000 setps AND the loss on the train is:1.458297610282898\n",
      "0.9344\n",
      "the 20500 setps AND the loss on the train is:1.490140438079834\n",
      "0.928\n",
      "the 21000 setps AND the loss on the train is:1.6362149715423584\n",
      "0.932\n",
      "the 21500 setps AND the loss on the train is:1.6098356246948242\n",
      "0.9261\n",
      "the 22000 setps AND the loss on the train is:1.5382232666015625\n",
      "0.9251\n",
      "the 22500 setps AND the loss on the train is:1.4722859859466553\n",
      "0.9296\n",
      "the 23000 setps AND the loss on the train is:1.5014903545379639\n",
      "0.9312\n",
      "the 23500 setps AND the loss on the train is:1.5875327587127686\n",
      "0.9277\n",
      "the 24000 setps AND the loss on the train is:1.5551061630249023\n",
      "0.934\n",
      "the 24500 setps AND the loss on the train is:1.5246531963348389\n",
      "0.9279\n",
      "the 25000 setps AND the loss on the train is:1.538106918334961\n",
      "0.9162\n",
      "the 25500 setps AND the loss on the train is:1.538568139076233\n",
      "0.9357\n",
      "the 26000 setps AND the loss on the train is:1.5529171228408813\n",
      "0.9316\n",
      "the 26500 setps AND the loss on the train is:1.5876030921936035\n",
      "0.9369\n",
      "the 27000 setps AND the loss on the train is:1.580548644065857\n",
      "0.9336\n",
      "the 27500 setps AND the loss on the train is:1.5271987915039062\n",
      "0.9372\n",
      "the 28000 setps AND the loss on the train is:1.5165438652038574\n",
      "0.936\n",
      "the 28500 setps AND the loss on the train is:1.5262356996536255\n",
      "0.9308\n",
      "the 29000 setps AND the loss on the train is:1.473811388015747\n",
      "0.9347\n",
      "the 29500 setps AND the loss on the train is:1.714573860168457\n",
      "0.9224\n",
      "the 30000 setps AND the loss on the train is:1.6019389629364014\n",
      "0.9359\n",
      "the 30500 setps AND the loss on the train is:1.5097429752349854\n",
      "0.9297\n",
      "the 31000 setps AND the loss on the train is:1.5857040882110596\n",
      "0.9304\n",
      "the 31500 setps AND the loss on the train is:1.5190290212631226\n",
      "0.9235\n",
      "the 32000 setps AND the loss on the train is:1.6001102924346924\n",
      "0.9356\n",
      "the 32500 setps AND the loss on the train is:1.5830031633377075\n",
      "0.9244\n",
      "the 33000 setps AND the loss on the train is:1.4501160383224487\n",
      "0.9325\n",
      "the 33500 setps AND the loss on the train is:1.5222175121307373\n",
      "0.9226\n",
      "the 34000 setps AND the loss on the train is:1.5044571161270142\n",
      "0.9362\n",
      "the 34500 setps AND the loss on the train is:1.4989928007125854\n",
      "0.9155\n",
      "the 35000 setps AND the loss on the train is:1.4918147325515747\n",
      "0.9266\n",
      "the 35500 setps AND the loss on the train is:1.4999380111694336\n",
      "0.9403\n",
      "the 36000 setps AND the loss on the train is:1.5754839181900024\n",
      "0.9357\n",
      "the 36500 setps AND the loss on the train is:1.5021812915802002\n",
      "0.9335\n",
      "the 37000 setps AND the loss on the train is:1.4686614274978638\n",
      "0.9303\n",
      "the 37500 setps AND the loss on the train is:1.506102204322815\n",
      "0.9262\n",
      "the 38000 setps AND the loss on the train is:1.4289766550064087\n",
      "0.934\n",
      "the 38500 setps AND the loss on the train is:1.5746711492538452\n",
      "0.9336\n",
      "the 39000 setps AND the loss on the train is:1.4944888353347778\n",
      "0.9295\n",
      "the 39500 setps AND the loss on the train is:1.5175437927246094\n",
      "0.9272\n",
      "the 40000 setps AND the loss on the train is:1.4751697778701782\n",
      "0.9345\n",
      "the 40500 setps AND the loss on the train is:1.6462876796722412\n",
      "0.9388\n",
      "the 41000 setps AND the loss on the train is:1.436800479888916\n",
      "0.9363\n",
      "the 41500 setps AND the loss on the train is:1.5557419061660767\n",
      "0.9303\n",
      "the 42000 setps AND the loss on the train is:1.4653030633926392\n",
      "0.9318\n",
      "the 42500 setps AND the loss on the train is:1.4556492567062378\n",
      "0.934\n",
      "the 43000 setps AND the loss on the train is:1.4819869995117188\n",
      "0.9321\n",
      "the 43500 setps AND the loss on the train is:1.4557509422302246\n",
      "0.9329\n",
      "the 44000 setps AND the loss on the train is:1.5397052764892578\n",
      "0.9343\n",
      "the 44500 setps AND the loss on the train is:1.5728365182876587\n",
      "0.918\n",
      "the 45000 setps AND the loss on the train is:1.465397834777832\n",
      "0.9321\n",
      "the 45500 setps AND the loss on the train is:1.4944151639938354\n",
      "0.9324\n",
      "the 46000 setps AND the loss on the train is:1.5663520097732544\n",
      "0.9292\n",
      "the 46500 setps AND the loss on the train is:1.5371792316436768\n",
      "0.928\n",
      "the 47000 setps AND the loss on the train is:1.53759765625\n",
      "0.9318\n",
      "the 47500 setps AND the loss on the train is:1.463387370109558\n",
      "0.9314\n",
      "the 48000 setps AND the loss on the train is:1.543076515197754\n",
      "0.9358\n",
      "the 48500 setps AND the loss on the train is:1.5673538446426392\n",
      "0.9303\n",
      "the 49000 setps AND the loss on the train is:1.4344779253005981\n",
      "0.9364\n",
      "the 49500 setps AND the loss on the train is:1.6033961772918701\n",
      "0.9302\n",
      "the 50000 setps AND the loss on the train is:1.4870858192443848\n",
      "0.9344\n",
      "the 50500 setps AND the loss on the train is:1.6196255683898926\n",
      "0.9164\n",
      "the 51000 setps AND the loss on the train is:1.5125067234039307\n",
      "0.9362\n",
      "the 51500 setps AND the loss on the train is:1.4267388582229614\n",
      "0.9366\n",
      "the 52000 setps AND the loss on the train is:1.562127709388733\n",
      "0.9289\n",
      "the 52500 setps AND the loss on the train is:1.5965393781661987\n",
      "0.9404\n",
      "the 53000 setps AND the loss on the train is:1.544497013092041\n",
      "0.9248\n",
      "the 53500 setps AND the loss on the train is:1.5049071311950684\n",
      "0.9351\n",
      "the 54000 setps AND the loss on the train is:1.5086216926574707\n",
      "0.9378\n",
      "the 54500 setps AND the loss on the train is:1.5537853240966797\n",
      "0.9345\n",
      "the 55000 setps AND the loss on the train is:1.4780616760253906\n",
      "0.9348\n",
      "the 55500 setps AND the loss on the train is:1.5467379093170166\n",
      "0.9273\n",
      "the 56000 setps AND the loss on the train is:1.4511520862579346\n",
      "0.931\n",
      "the 56500 setps AND the loss on the train is:1.509765386581421\n",
      "0.9277\n",
      "the 57000 setps AND the loss on the train is:1.5275099277496338\n",
      "0.9366\n",
      "the 57500 setps AND the loss on the train is:1.4613854885101318\n",
      "0.9357\n",
      "the 58000 setps AND the loss on the train is:1.5000859498977661\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9289\n",
      "the 58500 setps AND the loss on the train is:1.4430203437805176\n",
      "0.9381\n",
      "the 59000 setps AND the loss on the train is:1.476669430732727\n",
      "0.9337\n",
      "the 59500 setps AND the loss on the train is:1.5377986431121826\n",
      "0.9341\n",
      "the 60000 setps AND the loss on the train is:1.5071585178375244\n",
      "0.9328\n",
      "the 60500 setps AND the loss on the train is:1.553735613822937\n",
      "0.9358\n",
      "the 61000 setps AND the loss on the train is:1.5456042289733887\n",
      "0.9331\n",
      "the 61500 setps AND the loss on the train is:1.4960187673568726\n",
      "0.9326\n",
      "the 62000 setps AND the loss on the train is:1.516965627670288\n",
      "0.9377\n",
      "the 62500 setps AND the loss on the train is:1.5136793851852417\n",
      "0.9301\n",
      "the 63000 setps AND the loss on the train is:1.5238864421844482\n",
      "0.9356\n",
      "the 63500 setps AND the loss on the train is:1.6100573539733887\n",
      "0.9208\n",
      "the 64000 setps AND the loss on the train is:1.4475505352020264\n",
      "0.9362\n",
      "the 64500 setps AND the loss on the train is:1.478169322013855\n",
      "0.9338\n",
      "the 65000 setps AND the loss on the train is:1.5089353322982788\n",
      "0.933\n",
      "the 65500 setps AND the loss on the train is:1.5392588376998901\n",
      "0.935\n",
      "the 66000 setps AND the loss on the train is:1.510608434677124\n",
      "0.9346\n",
      "the 66500 setps AND the loss on the train is:1.4119797945022583\n",
      "0.9327\n",
      "the 67000 setps AND the loss on the train is:1.5435454845428467\n",
      "0.9314\n",
      "the 67500 setps AND the loss on the train is:1.528686761856079\n",
      "0.9348\n",
      "the 68000 setps AND the loss on the train is:1.4949395656585693\n",
      "0.9363\n",
      "the 68500 setps AND the loss on the train is:1.6162165403366089\n",
      "0.9345\n",
      "the 69000 setps AND the loss on the train is:1.5212582349777222\n",
      "0.9345\n",
      "the 69500 setps AND the loss on the train is:1.4958341121673584\n",
      "0.9359\n",
      "the 70000 setps AND the loss on the train is:1.5180898904800415\n",
      "0.9352\n",
      "the 70500 setps AND the loss on the train is:1.4928358793258667\n",
      "0.9363\n",
      "the 71000 setps AND the loss on the train is:1.4854683876037598\n",
      "0.9342\n",
      "the 71500 setps AND the loss on the train is:1.5030004978179932\n",
      "0.9368\n",
      "the 72000 setps AND the loss on the train is:1.5187101364135742\n",
      "0.9315\n",
      "the 72500 setps AND the loss on the train is:1.5573549270629883\n",
      "0.9313\n",
      "the 73000 setps AND the loss on the train is:1.4641103744506836\n",
      "0.933\n",
      "the 73500 setps AND the loss on the train is:1.4656410217285156\n",
      "0.934\n",
      "the 74000 setps AND the loss on the train is:1.452636957168579\n",
      "0.9357\n",
      "the 74500 setps AND the loss on the train is:1.5498093366622925\n",
      "0.9371\n",
      "the 75000 setps AND the loss on the train is:1.5829578638076782\n",
      "0.93\n",
      "the 75500 setps AND the loss on the train is:1.5527639389038086\n",
      "0.9353\n",
      "the 76000 setps AND the loss on the train is:1.5149762630462646\n",
      "0.9341\n",
      "the 76500 setps AND the loss on the train is:1.5390185117721558\n",
      "0.9301\n",
      "the 77000 setps AND the loss on the train is:1.4843449592590332\n",
      "0.9358\n",
      "the 77500 setps AND the loss on the train is:1.481234073638916\n",
      "0.9312\n",
      "the 78000 setps AND the loss on the train is:1.493485927581787\n",
      "0.93\n",
      "the 78500 setps AND the loss on the train is:1.577013611793518\n",
      "0.9333\n",
      "the 79000 setps AND the loss on the train is:1.5027827024459839\n",
      "0.9332\n",
      "the 79500 setps AND the loss on the train is:1.6946496963500977\n",
      "0.9163\n",
      "the 80000 setps AND the loss on the train is:1.5284982919692993\n",
      "0.9357\n",
      "the 80500 setps AND the loss on the train is:1.6553540229797363\n",
      "0.9364\n",
      "the 81000 setps AND the loss on the train is:1.500757098197937\n",
      "0.9371\n",
      "the 81500 setps AND the loss on the train is:1.6310583353042603\n",
      "0.9301\n",
      "the 82000 setps AND the loss on the train is:1.5647659301757812\n",
      "0.9362\n",
      "the 82500 setps AND the loss on the train is:1.4771790504455566\n",
      "0.9348\n",
      "the 83000 setps AND the loss on the train is:1.5488085746765137\n",
      "0.9307\n",
      "the 83500 setps AND the loss on the train is:1.5776515007019043\n",
      "0.933\n",
      "the 84000 setps AND the loss on the train is:1.5168615579605103\n",
      "0.9356\n",
      "the 84500 setps AND the loss on the train is:1.4638139009475708\n",
      "0.9378\n",
      "the 85000 setps AND the loss on the train is:1.4867613315582275\n",
      "0.932\n",
      "the 85500 setps AND the loss on the train is:1.6463792324066162\n",
      "0.9347\n",
      "the 86000 setps AND the loss on the train is:1.4832210540771484\n",
      "0.935\n",
      "the 86500 setps AND the loss on the train is:1.6666303873062134\n",
      "0.9284\n",
      "the 87000 setps AND the loss on the train is:1.5025912523269653\n",
      "0.9343\n",
      "the 87500 setps AND the loss on the train is:1.5312377214431763\n",
      "0.9339\n",
      "the 88000 setps AND the loss on the train is:1.541454553604126\n",
      "0.9311\n",
      "the 88500 setps AND the loss on the train is:1.5252822637557983\n",
      "0.9327\n",
      "the 89000 setps AND the loss on the train is:1.5233888626098633\n",
      "0.9334\n",
      "the 89500 setps AND the loss on the train is:1.4934405088424683\n",
      "0.9329\n",
      "the 90000 setps AND the loss on the train is:1.4545974731445312\n",
      "0.9303\n",
      "the 90500 setps AND the loss on the train is:1.525895357131958\n",
      "0.9371\n",
      "the 91000 setps AND the loss on the train is:1.60105299949646\n",
      "0.9356\n",
      "the 91500 setps AND the loss on the train is:1.4882978200912476\n",
      "0.9355\n",
      "the 92000 setps AND the loss on the train is:1.4913145303726196\n",
      "0.9267\n",
      "the 92500 setps AND the loss on the train is:1.5773197412490845\n",
      "0.9291\n",
      "the 93000 setps AND the loss on the train is:1.4765207767486572\n",
      "0.9334\n",
      "the 93500 setps AND the loss on the train is:1.5151550769805908\n",
      "0.9372\n",
      "the 94000 setps AND the loss on the train is:1.5190105438232422\n",
      "0.9302\n",
      "the 94500 setps AND the loss on the train is:1.5480507612228394\n",
      "0.931\n",
      "the 95000 setps AND the loss on the train is:1.4792170524597168\n",
      "0.9389\n",
      "the 95500 setps AND the loss on the train is:1.5337233543395996\n",
      "0.9361\n",
      "the 96000 setps AND the loss on the train is:1.5131639242172241\n",
      "0.9375\n",
      "the 96500 setps AND the loss on the train is:1.6198089122772217\n",
      "0.9359\n",
      "the 97000 setps AND the loss on the train is:1.5173171758651733\n",
      "0.9337\n",
      "the 97500 setps AND the loss on the train is:1.651485562324524\n",
      "0.9194\n",
      "the 98000 setps AND the loss on the train is:1.540311574935913\n",
      "0.9353\n",
      "the 98500 setps AND the loss on the train is:1.5115548372268677\n",
      "0.9378\n",
      "the 99000 setps AND the loss on the train is:1.5569194555282593\n",
      "0.9188\n",
      "the 99500 setps AND the loss on the train is:1.4747898578643799\n",
      "0.9294\n",
      "the 100000 setps AND the loss on the train is:1.4804154634475708\n",
      "0.9384\n",
      "the 100500 setps AND the loss on the train is:1.4838123321533203\n",
      "0.9358\n",
      "the 101000 setps AND the loss on the train is:1.4930477142333984\n",
      "0.9355\n",
      "the 101500 setps AND the loss on the train is:1.4621496200561523\n",
      "0.9323\n",
      "the 102000 setps AND the loss on the train is:1.4439369440078735\n",
      "0.9352\n",
      "the 102500 setps AND the loss on the train is:1.496561050415039\n",
      "0.9373\n",
      "the 103000 setps AND the loss on the train is:1.5087453126907349\n",
      "0.9342\n",
      "the 103500 setps AND the loss on the train is:1.5068225860595703\n",
      "0.9363\n",
      "the 104000 setps AND the loss on the train is:1.5094408988952637\n",
      "0.931\n",
      "the 104500 setps AND the loss on the train is:1.4913562536239624\n",
      "0.9338\n",
      "the 105000 setps AND the loss on the train is:1.5814610719680786\n",
      "0.9353\n",
      "the 105500 setps AND the loss on the train is:1.5374464988708496\n",
      "0.9317\n",
      "the 106000 setps AND the loss on the train is:1.5130963325500488\n",
      "0.9361\n",
      "the 106500 setps AND the loss on the train is:1.610314130783081\n",
      "0.9356\n",
      "the 107000 setps AND the loss on the train is:1.5156171321868896\n",
      "0.9335\n",
      "the 107500 setps AND the loss on the train is:1.5409518480300903\n",
      "0.9336\n",
      "the 108000 setps AND the loss on the train is:1.502416968345642\n",
      "0.9369\n",
      "the 108500 setps AND the loss on the train is:1.4936009645462036\n",
      "0.9326\n",
      "the 109000 setps AND the loss on the train is:1.5195367336273193\n",
      "0.9329\n",
      "the 109500 setps AND the loss on the train is:1.6497658491134644\n",
      "0.936\n",
      "the 110000 setps AND the loss on the train is:1.541998028755188\n",
      "0.939\n",
      "the 110500 setps AND the loss on the train is:1.498409628868103\n",
      "0.9339\n",
      "the 111000 setps AND the loss on the train is:1.5109074115753174\n",
      "0.9374\n",
      "the 111500 setps AND the loss on the train is:1.5524505376815796\n",
      "0.9291\n",
      "the 112000 setps AND the loss on the train is:1.5287436246871948\n",
      "0.9336\n",
      "the 112500 setps AND the loss on the train is:1.5093172788619995\n",
      "0.9376\n",
      "the 113000 setps AND the loss on the train is:1.4303423166275024\n",
      "0.9298\n",
      "the 113500 setps AND the loss on the train is:1.5169451236724854\n",
      "0.9325\n",
      "the 114000 setps AND the loss on the train is:1.5488961935043335\n",
      "0.9358\n",
      "the 114500 setps AND the loss on the train is:1.4741100072860718\n",
      "0.929\n",
      "the 115000 setps AND the loss on the train is:1.5242469310760498\n",
      "0.9282\n",
      "the 115500 setps AND the loss on the train is:1.5031025409698486\n",
      "0.9353\n",
      "the 116000 setps AND the loss on the train is:1.4557132720947266\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9351\n",
      "the 116500 setps AND the loss on the train is:1.5781550407409668\n",
      "0.9284\n",
      "the 117000 setps AND the loss on the train is:1.447045922279358\n",
      "0.9329\n",
      "the 117500 setps AND the loss on the train is:1.568008303642273\n",
      "0.9366\n",
      "the 118000 setps AND the loss on the train is:1.5851284265518188\n",
      "0.9356\n",
      "the 118500 setps AND the loss on the train is:1.4957647323608398\n",
      "0.9271\n",
      "the 119000 setps AND the loss on the train is:1.4827145338058472\n",
      "0.9335\n",
      "the 119500 setps AND the loss on the train is:1.5943365097045898\n",
      "0.9328\n",
      "the 120000 setps AND the loss on the train is:1.5128521919250488\n",
      "0.9295\n",
      "the 120500 setps AND the loss on the train is:1.557506799697876\n",
      "0.9352\n",
      "the 121000 setps AND the loss on the train is:1.5239267349243164\n",
      "0.9386\n",
      "the 121500 setps AND the loss on the train is:1.505409836769104\n",
      "0.9362\n",
      "the 122000 setps AND the loss on the train is:1.4856384992599487\n",
      "0.9353\n",
      "the 122500 setps AND the loss on the train is:1.5856895446777344\n",
      "0.9343\n",
      "the 123000 setps AND the loss on the train is:1.562258005142212\n",
      "0.9365\n",
      "the 123500 setps AND the loss on the train is:1.519318699836731\n",
      "0.9258\n",
      "the 124000 setps AND the loss on the train is:1.5367679595947266\n",
      "0.9367\n",
      "the 124500 setps AND the loss on the train is:1.4562876224517822\n",
      "0.9344\n",
      "the 125000 setps AND the loss on the train is:1.5377566814422607\n",
      "0.936\n",
      "the 125500 setps AND the loss on the train is:1.549912691116333\n",
      "0.9323\n",
      "the 126000 setps AND the loss on the train is:1.5107485055923462\n",
      "0.9333\n",
      "the 126500 setps AND the loss on the train is:1.5105066299438477\n",
      "0.938\n",
      "the 127000 setps AND the loss on the train is:1.5101375579833984\n",
      "0.9363\n",
      "the 127500 setps AND the loss on the train is:1.5548291206359863\n",
      "0.9328\n",
      "the 128000 setps AND the loss on the train is:1.5533312559127808\n",
      "0.9393\n",
      "the 128500 setps AND the loss on the train is:1.5121359825134277\n",
      "0.9339\n",
      "the 129000 setps AND the loss on the train is:1.5366853475570679\n",
      "0.9349\n",
      "the 129500 setps AND the loss on the train is:1.4819419384002686\n",
      "0.933\n",
      "the 130000 setps AND the loss on the train is:1.6614457368850708\n",
      "0.9356\n",
      "the 130500 setps AND the loss on the train is:1.5500781536102295\n",
      "0.9348\n",
      "the 131000 setps AND the loss on the train is:1.4700030088424683\n",
      "0.9354\n",
      "the 131500 setps AND the loss on the train is:1.480562686920166\n",
      "0.9357\n",
      "the 132000 setps AND the loss on the train is:1.601151466369629\n",
      "0.9227\n",
      "the 132500 setps AND the loss on the train is:1.4517626762390137\n",
      "0.9344\n",
      "the 133000 setps AND the loss on the train is:1.4907530546188354\n",
      "0.9384\n",
      "the 133500 setps AND the loss on the train is:1.614253282546997\n",
      "0.935\n",
      "the 134000 setps AND the loss on the train is:1.59430730342865\n",
      "0.9269\n",
      "the 134500 setps AND the loss on the train is:1.5343163013458252\n",
      "0.9345\n",
      "the 135000 setps AND the loss on the train is:1.5418630838394165\n",
      "0.931\n",
      "the 135500 setps AND the loss on the train is:1.4818445444107056\n",
      "0.9354\n",
      "the 136000 setps AND the loss on the train is:1.5334571599960327\n",
      "0.9345\n",
      "the 136500 setps AND the loss on the train is:1.5698387622833252\n",
      "0.9395\n",
      "the 137000 setps AND the loss on the train is:1.5841050148010254\n",
      "0.928\n",
      "the 137500 setps AND the loss on the train is:1.5148093700408936\n",
      "0.9382\n",
      "the 138000 setps AND the loss on the train is:1.4446570873260498\n",
      "0.9253\n",
      "the 138500 setps AND the loss on the train is:1.5539829730987549\n",
      "0.9281\n",
      "the 139000 setps AND the loss on the train is:1.4402570724487305\n",
      "0.9372\n",
      "the 139500 setps AND the loss on the train is:1.4975237846374512\n",
      "0.9321\n",
      "the 140000 setps AND the loss on the train is:1.4797567129135132\n",
      "0.935\n",
      "the 140500 setps AND the loss on the train is:1.48799729347229\n",
      "0.9348\n",
      "the 141000 setps AND the loss on the train is:1.4454503059387207\n",
      "0.9339\n",
      "the 141500 setps AND the loss on the train is:1.5699212551116943\n",
      "0.9263\n",
      "the 142000 setps AND the loss on the train is:1.4887096881866455\n",
      "0.9374\n",
      "the 142500 setps AND the loss on the train is:1.534642219543457\n",
      "0.9356\n",
      "the 143000 setps AND the loss on the train is:1.5305812358856201\n",
      "0.9364\n",
      "the 143500 setps AND the loss on the train is:1.5000596046447754\n",
      "0.9336\n",
      "the 144000 setps AND the loss on the train is:1.5352911949157715\n",
      "0.9263\n",
      "the 144500 setps AND the loss on the train is:1.5893323421478271\n",
      "0.9369\n",
      "the 145000 setps AND the loss on the train is:1.4937107563018799\n",
      "0.9381\n",
      "the 145500 setps AND the loss on the train is:1.5085705518722534\n",
      "0.9351\n",
      "the 146000 setps AND the loss on the train is:1.5545916557312012\n",
      "0.9355\n",
      "the 146500 setps AND the loss on the train is:1.5188111066818237\n",
      "0.937\n",
      "the 147000 setps AND the loss on the train is:1.453755259513855\n",
      "0.9339\n",
      "the 147500 setps AND the loss on the train is:1.4859040975570679\n",
      "0.9367\n",
      "the 148000 setps AND the loss on the train is:1.5850569009780884\n",
      "0.9303\n",
      "the 148500 setps AND the loss on the train is:1.446597933769226\n",
      "0.9312\n",
      "the 149000 setps AND the loss on the train is:1.5458579063415527\n",
      "0.935\n",
      "the 149500 setps AND the loss on the train is:1.5265346765518188\n",
      "0.9261\n",
      "the 150000 setps AND the loss on the train is:1.6088757514953613\n",
      "0.9322\n",
      "the 150500 setps AND the loss on the train is:1.5321650505065918\n",
      "0.9355\n",
      "the 151000 setps AND the loss on the train is:1.5629453659057617\n",
      "0.9133\n",
      "the 151500 setps AND the loss on the train is:1.4955657720565796\n",
      "0.9347\n",
      "the 152000 setps AND the loss on the train is:1.511465072631836\n",
      "0.9293\n",
      "the 152500 setps AND the loss on the train is:1.4801043272018433\n",
      "0.9339\n",
      "the 153000 setps AND the loss on the train is:1.593002438545227\n",
      "0.9303\n",
      "the 153500 setps AND the loss on the train is:1.5515187978744507\n",
      "0.9358\n",
      "the 154000 setps AND the loss on the train is:1.5254942178726196\n",
      "0.9308\n",
      "the 154500 setps AND the loss on the train is:1.4861267805099487\n",
      "0.9372\n",
      "the 155000 setps AND the loss on the train is:1.5207902193069458\n",
      "0.9363\n",
      "the 155500 setps AND the loss on the train is:1.5290758609771729\n",
      "0.9363\n",
      "the 156000 setps AND the loss on the train is:1.4484992027282715\n",
      "0.9349\n",
      "the 156500 setps AND the loss on the train is:1.5320450067520142\n",
      "0.9372\n",
      "the 157000 setps AND the loss on the train is:1.5115776062011719\n",
      "0.9347\n",
      "the 157500 setps AND the loss on the train is:1.495070219039917\n",
      "0.9374\n",
      "the 158000 setps AND the loss on the train is:1.6016660928726196\n",
      "0.9377\n",
      "the 158500 setps AND the loss on the train is:1.5377219915390015\n",
      "0.9394\n",
      "the 159000 setps AND the loss on the train is:1.5236753225326538\n",
      "0.9334\n",
      "the 159500 setps AND the loss on the train is:1.5595046281814575\n",
      "0.9349\n",
      "the 160000 setps AND the loss on the train is:1.5586199760437012\n",
      "0.9354\n",
      "the 160500 setps AND the loss on the train is:1.459303617477417\n",
      "0.9312\n",
      "the 161000 setps AND the loss on the train is:1.462157964706421\n",
      "0.9366\n",
      "the 161500 setps AND the loss on the train is:1.6077880859375\n",
      "0.9344\n",
      "the 162000 setps AND the loss on the train is:1.4374302625656128\n",
      "0.9363\n",
      "the 162500 setps AND the loss on the train is:1.4760658740997314\n",
      "0.9368\n",
      "the 163000 setps AND the loss on the train is:1.4733681678771973\n",
      "0.9387\n",
      "the 163500 setps AND the loss on the train is:1.4450092315673828\n",
      "0.9318\n",
      "the 164000 setps AND the loss on the train is:1.4936540126800537\n",
      "0.9344\n",
      "the 164500 setps AND the loss on the train is:1.6117123365402222\n",
      "0.937\n",
      "the 165000 setps AND the loss on the train is:1.4748891592025757\n",
      "0.9345\n",
      "the 165500 setps AND the loss on the train is:1.438654899597168\n",
      "0.9351\n",
      "the 166000 setps AND the loss on the train is:1.5216654539108276\n",
      "0.9354\n",
      "the 166500 setps AND the loss on the train is:1.4999874830245972\n",
      "0.9329\n",
      "the 167000 setps AND the loss on the train is:1.5806987285614014\n",
      "0.9262\n",
      "the 167500 setps AND the loss on the train is:1.5512343645095825\n",
      "0.9384\n",
      "the 168000 setps AND the loss on the train is:1.550986409187317\n",
      "0.9363\n",
      "the 168500 setps AND the loss on the train is:1.4314451217651367\n",
      "0.9349\n",
      "the 169000 setps AND the loss on the train is:1.4853367805480957\n",
      "0.9357\n",
      "the 169500 setps AND the loss on the train is:1.5280463695526123\n",
      "0.9354\n",
      "the 170000 setps AND the loss on the train is:1.493437647819519\n",
      "0.936\n",
      "the 170500 setps AND the loss on the train is:1.5053738355636597\n",
      "0.9304\n",
      "the 171000 setps AND the loss on the train is:1.4929828643798828\n",
      "0.9379\n",
      "the 171500 setps AND the loss on the train is:1.5829339027404785\n",
      "0.934\n",
      "the 172000 setps AND the loss on the train is:1.5461878776550293\n",
      "0.9346\n",
      "the 172500 setps AND the loss on the train is:1.5869123935699463\n",
      "0.9317\n",
      "the 173000 setps AND the loss on the train is:1.5031356811523438\n",
      "0.9295\n",
      "the 173500 setps AND the loss on the train is:1.5014760494232178\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9356\n",
      "the 174000 setps AND the loss on the train is:1.5162492990493774\n",
      "0.9289\n",
      "the 174500 setps AND the loss on the train is:1.529920220375061\n",
      "0.9357\n",
      "the 175000 setps AND the loss on the train is:1.4662854671478271\n",
      "0.9383\n",
      "the 175500 setps AND the loss on the train is:1.625370979309082\n",
      "0.9342\n",
      "the 176000 setps AND the loss on the train is:1.51829993724823\n",
      "0.9275\n",
      "the 176500 setps AND the loss on the train is:1.5628669261932373\n",
      "0.9337\n",
      "the 177000 setps AND the loss on the train is:1.4220448732376099\n",
      "0.9346\n",
      "the 177500 setps AND the loss on the train is:1.5777846574783325\n",
      "0.9343\n",
      "the 178000 setps AND the loss on the train is:1.491089940071106\n",
      "0.9371\n",
      "the 178500 setps AND the loss on the train is:1.515312671661377\n",
      "0.9327\n",
      "the 179000 setps AND the loss on the train is:1.5022624731063843\n",
      "0.9353\n",
      "the 179500 setps AND the loss on the train is:1.4747486114501953\n",
      "0.9339\n",
      "the 180000 setps AND the loss on the train is:1.5158920288085938\n",
      "0.9288\n",
      "the 180500 setps AND the loss on the train is:1.525439739227295\n",
      "0.9376\n",
      "the 181000 setps AND the loss on the train is:1.5157065391540527\n",
      "0.9357\n",
      "the 181500 setps AND the loss on the train is:1.6000725030899048\n",
      "0.9382\n",
      "the 182000 setps AND the loss on the train is:1.5110055208206177\n",
      "0.9354\n",
      "the 182500 setps AND the loss on the train is:1.5272846221923828\n",
      "0.9349\n",
      "the 183000 setps AND the loss on the train is:1.5099072456359863\n",
      "0.9345\n",
      "the 183500 setps AND the loss on the train is:1.4728323221206665\n",
      "0.9287\n",
      "the 184000 setps AND the loss on the train is:1.5467581748962402\n",
      "0.9338\n",
      "the 184500 setps AND the loss on the train is:1.5166798830032349\n",
      "0.9333\n",
      "the 185000 setps AND the loss on the train is:1.4935203790664673\n",
      "0.9379\n",
      "the 185500 setps AND the loss on the train is:1.595328450202942\n",
      "0.9337\n",
      "the 186000 setps AND the loss on the train is:1.4859950542449951\n",
      "0.9347\n",
      "the 186500 setps AND the loss on the train is:1.528149127960205\n",
      "0.9331\n",
      "the 187000 setps AND the loss on the train is:1.4643747806549072\n",
      "0.933\n",
      "the 187500 setps AND the loss on the train is:1.5833518505096436\n",
      "0.9333\n",
      "the 188000 setps AND the loss on the train is:1.478362798690796\n",
      "0.9355\n",
      "the 188500 setps AND the loss on the train is:1.551778793334961\n",
      "0.9348\n",
      "the 189000 setps AND the loss on the train is:1.5501704216003418\n",
      "0.9365\n",
      "the 189500 setps AND the loss on the train is:1.6859910488128662\n",
      "0.9347\n",
      "the 190000 setps AND the loss on the train is:1.4708852767944336\n",
      "0.9331\n",
      "the 190500 setps AND the loss on the train is:1.481413722038269\n",
      "0.9352\n",
      "the 191000 setps AND the loss on the train is:1.4927362203598022\n",
      "0.9373\n",
      "the 191500 setps AND the loss on the train is:1.5056047439575195\n",
      "0.933\n",
      "the 192000 setps AND the loss on the train is:1.4537335634231567\n",
      "0.9368\n",
      "the 192500 setps AND the loss on the train is:1.474960207939148\n",
      "0.9338\n",
      "the 193000 setps AND the loss on the train is:1.5847935676574707\n",
      "0.9376\n",
      "the 193500 setps AND the loss on the train is:1.5701712369918823\n",
      "0.9335\n",
      "the 194000 setps AND the loss on the train is:1.5436993837356567\n",
      "0.9362\n",
      "the 194500 setps AND the loss on the train is:1.4553478956222534\n",
      "0.9363\n",
      "the 195000 setps AND the loss on the train is:1.587643027305603\n",
      "0.9328\n",
      "the 195500 setps AND the loss on the train is:1.5140495300292969\n",
      "0.9378\n",
      "the 196000 setps AND the loss on the train is:1.4703680276870728\n",
      "0.9367\n",
      "the 196500 setps AND the loss on the train is:1.5534001588821411\n",
      "0.9307\n",
      "the 197000 setps AND the loss on the train is:1.4997881650924683\n",
      "0.9283\n",
      "the 197500 setps AND the loss on the train is:1.5120400190353394\n",
      "0.9353\n",
      "the 198000 setps AND the loss on the train is:1.561081886291504\n",
      "0.9343\n",
      "the 198500 setps AND the loss on the train is:1.6196902990341187\n",
      "0.9343\n",
      "the 199000 setps AND the loss on the train is:1.5786937475204468\n",
      "0.9305\n",
      "the 199500 setps AND the loss on the train is:1.4396140575408936\n",
      "0.937\n",
      "the 200000 setps AND the loss on the train is:1.5847241878509521\n",
      "0.938\n",
      "the 200500 setps AND the loss on the train is:1.508503794670105\n",
      "0.9367\n",
      "the 201000 setps AND the loss on the train is:1.5481746196746826\n",
      "0.9333\n",
      "the 201500 setps AND the loss on the train is:1.459315538406372\n",
      "0.9349\n",
      "the 202000 setps AND the loss on the train is:1.5673917531967163\n",
      "0.9361\n",
      "the 202500 setps AND the loss on the train is:1.531393051147461\n",
      "0.933\n",
      "the 203000 setps AND the loss on the train is:1.589592456817627\n",
      "0.9373\n",
      "the 203500 setps AND the loss on the train is:1.5628879070281982\n",
      "0.9317\n",
      "the 204000 setps AND the loss on the train is:1.5744407176971436\n",
      "0.9349\n",
      "the 204500 setps AND the loss on the train is:1.5142773389816284\n",
      "0.9344\n",
      "the 205000 setps AND the loss on the train is:1.5024888515472412\n",
      "0.9367\n",
      "the 205500 setps AND the loss on the train is:1.577832579612732\n",
      "0.934\n",
      "the 206000 setps AND the loss on the train is:1.5370069742202759\n",
      "0.9363\n",
      "the 206500 setps AND the loss on the train is:1.6074023246765137\n",
      "0.9293\n",
      "the 207000 setps AND the loss on the train is:1.545634150505066\n",
      "0.9352\n",
      "the 207500 setps AND the loss on the train is:1.507816195487976\n",
      "0.9315\n",
      "the 208000 setps AND the loss on the train is:1.480644702911377\n",
      "0.9356\n",
      "the 208500 setps AND the loss on the train is:1.6948553323745728\n",
      "0.9387\n",
      "the 209000 setps AND the loss on the train is:1.4340229034423828\n",
      "0.9351\n",
      "the 209500 setps AND the loss on the train is:1.49020516872406\n",
      "0.9348\n",
      "the 210000 setps AND the loss on the train is:1.4593043327331543\n",
      "0.9317\n",
      "the 210500 setps AND the loss on the train is:1.4773168563842773\n",
      "0.9351\n",
      "the 211000 setps AND the loss on the train is:1.527359962463379\n",
      "0.9383\n",
      "the 211500 setps AND the loss on the train is:1.4978357553482056\n",
      "0.9382\n",
      "the 212000 setps AND the loss on the train is:1.499896764755249\n",
      "0.9347\n",
      "the 212500 setps AND the loss on the train is:1.4708560705184937\n",
      "0.9333\n",
      "the 213000 setps AND the loss on the train is:1.455500841140747\n",
      "0.9359\n",
      "the 213500 setps AND the loss on the train is:1.5389790534973145\n",
      "0.9372\n",
      "the 214000 setps AND the loss on the train is:1.4649735689163208\n",
      "0.9372\n",
      "the 214500 setps AND the loss on the train is:1.4618182182312012\n",
      "0.9379\n",
      "the 215000 setps AND the loss on the train is:1.554427981376648\n",
      "0.9374\n",
      "the 215500 setps AND the loss on the train is:1.5362436771392822\n",
      "0.931\n",
      "the 216000 setps AND the loss on the train is:1.5767444372177124\n",
      "0.9385\n",
      "the 216500 setps AND the loss on the train is:1.58437180519104\n",
      "0.9317\n",
      "the 217000 setps AND the loss on the train is:1.4828088283538818\n",
      "0.9345\n",
      "the 217500 setps AND the loss on the train is:1.5678397417068481\n",
      "0.9364\n",
      "the 218000 setps AND the loss on the train is:1.4603657722473145\n",
      "0.9368\n",
      "the 218500 setps AND the loss on the train is:1.4550056457519531\n",
      "0.9365\n",
      "the 219000 setps AND the loss on the train is:1.5200836658477783\n",
      "0.9358\n",
      "the 219500 setps AND the loss on the train is:1.4828590154647827\n",
      "0.9356\n",
      "the 220000 setps AND the loss on the train is:1.5185948610305786\n",
      "0.9363\n",
      "the 220500 setps AND the loss on the train is:1.4349141120910645\n",
      "0.933\n",
      "the 221000 setps AND the loss on the train is:1.4499670267105103\n",
      "0.9387\n",
      "the 221500 setps AND the loss on the train is:1.5159446001052856\n",
      "0.9388\n",
      "the 222000 setps AND the loss on the train is:1.577347755432129\n",
      "0.9319\n",
      "the 222500 setps AND the loss on the train is:1.489959955215454\n",
      "0.9341\n",
      "the 223000 setps AND the loss on the train is:1.4814515113830566\n",
      "0.9342\n",
      "the 223500 setps AND the loss on the train is:1.5220928192138672\n",
      "0.9343\n",
      "the 224000 setps AND the loss on the train is:1.4683582782745361\n",
      "0.9363\n",
      "the 224500 setps AND the loss on the train is:1.5453821420669556\n",
      "0.9341\n",
      "the 225000 setps AND the loss on the train is:1.5338085889816284\n",
      "0.9367\n",
      "the 225500 setps AND the loss on the train is:1.4783893823623657\n",
      "0.9346\n",
      "the 226000 setps AND the loss on the train is:1.4321436882019043\n",
      "0.9353\n",
      "the 226500 setps AND the loss on the train is:1.4987987279891968\n",
      "0.918\n",
      "the 227000 setps AND the loss on the train is:1.500112533569336\n",
      "0.9375\n",
      "the 227500 setps AND the loss on the train is:1.5438350439071655\n",
      "0.9324\n",
      "the 228000 setps AND the loss on the train is:1.4690660238265991\n",
      "0.9337\n",
      "the 228500 setps AND the loss on the train is:1.6193772554397583\n",
      "0.9262\n",
      "the 229000 setps AND the loss on the train is:1.505555272102356\n",
      "0.9375\n",
      "the 229500 setps AND the loss on the train is:1.547005534172058\n",
      "0.9352\n",
      "the 230000 setps AND the loss on the train is:1.4422072172164917\n",
      "0.9368\n",
      "the 230500 setps AND the loss on the train is:1.564345359802246\n",
      "0.9349\n",
      "the 231000 setps AND the loss on the train is:1.5947926044464111\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.931\n",
      "the 231500 setps AND the loss on the train is:1.5095481872558594\n",
      "0.9369\n",
      "the 232000 setps AND the loss on the train is:1.505771517753601\n",
      "0.9297\n",
      "the 232500 setps AND the loss on the train is:1.5586870908737183\n",
      "0.9382\n",
      "the 233000 setps AND the loss on the train is:1.4989378452301025\n",
      "0.9376\n",
      "the 233500 setps AND the loss on the train is:1.4313291311264038\n",
      "0.9337\n",
      "the 234000 setps AND the loss on the train is:1.4360110759735107\n",
      "0.9383\n",
      "the 234500 setps AND the loss on the train is:1.4999862909317017\n",
      "0.936\n",
      "the 235000 setps AND the loss on the train is:1.5273648500442505\n",
      "0.9369\n",
      "the 235500 setps AND the loss on the train is:1.4168840646743774\n",
      "0.9372\n",
      "the 236000 setps AND the loss on the train is:1.4881410598754883\n",
      "0.9382\n",
      "the 236500 setps AND the loss on the train is:1.4754588603973389\n",
      "0.937\n",
      "the 237000 setps AND the loss on the train is:1.546543002128601\n",
      "0.932\n",
      "the 237500 setps AND the loss on the train is:1.4648326635360718\n",
      "0.9359\n",
      "the 238000 setps AND the loss on the train is:1.6386326551437378\n",
      "0.9364\n",
      "the 238500 setps AND the loss on the train is:1.56855046749115\n",
      "0.9319\n",
      "the 239000 setps AND the loss on the train is:1.5787571668624878\n",
      "0.9378\n",
      "the 239500 setps AND the loss on the train is:1.6168467998504639\n",
      "0.9356\n",
      "the 240000 setps AND the loss on the train is:1.4413626194000244\n",
      "0.9354\n",
      "the 240500 setps AND the loss on the train is:1.4498118162155151\n",
      "0.9369\n",
      "the 241000 setps AND the loss on the train is:1.4495491981506348\n",
      "0.9373\n",
      "the 241500 setps AND the loss on the train is:1.489186406135559\n",
      "0.9345\n",
      "the 242000 setps AND the loss on the train is:1.480269432067871\n",
      "0.9345\n",
      "the 242500 setps AND the loss on the train is:1.4849109649658203\n",
      "0.9353\n",
      "the 243000 setps AND the loss on the train is:1.5344440937042236\n",
      "0.9337\n",
      "the 243500 setps AND the loss on the train is:1.515006422996521\n",
      "0.9332\n",
      "the 244000 setps AND the loss on the train is:1.5047987699508667\n",
      "0.9364\n",
      "the 244500 setps AND the loss on the train is:1.5236656665802002\n",
      "0.9309\n",
      "the 245000 setps AND the loss on the train is:1.5372309684753418\n",
      "0.9356\n",
      "the 245500 setps AND the loss on the train is:1.527786374092102\n",
      "0.9382\n",
      "the 246000 setps AND the loss on the train is:1.6181148290634155\n",
      "0.9299\n",
      "the 246500 setps AND the loss on the train is:1.6002068519592285\n",
      "0.9374\n",
      "the 247000 setps AND the loss on the train is:1.582763433456421\n",
      "0.9366\n",
      "the 247500 setps AND the loss on the train is:1.512193202972412\n",
      "0.9358\n",
      "the 248000 setps AND the loss on the train is:1.526210904121399\n",
      "0.9382\n",
      "the 248500 setps AND the loss on the train is:1.528878927230835\n",
      "0.9394\n",
      "the 249000 setps AND the loss on the train is:1.5095322132110596\n",
      "0.9342\n",
      "the 249500 setps AND the loss on the train is:1.482832670211792\n",
      "0.9333\n",
      "the 250000 setps AND the loss on the train is:1.525538444519043\n",
      "0.9338\n",
      "the 250500 setps AND the loss on the train is:1.5625035762786865\n",
      "0.9305\n",
      "the 251000 setps AND the loss on the train is:1.60440194606781\n",
      "0.9324\n",
      "the 251500 setps AND the loss on the train is:1.5527112483978271\n",
      "0.9375\n",
      "the 252000 setps AND the loss on the train is:1.5530781745910645\n",
      "0.9379\n",
      "the 252500 setps AND the loss on the train is:1.5492671728134155\n",
      "0.9301\n",
      "the 253000 setps AND the loss on the train is:1.5024921894073486\n",
      "0.9369\n",
      "the 253500 setps AND the loss on the train is:1.4261741638183594\n",
      "0.9373\n",
      "the 254000 setps AND the loss on the train is:1.6897449493408203\n",
      "0.935\n",
      "the 254500 setps AND the loss on the train is:1.5411059856414795\n",
      "0.9338\n",
      "the 255000 setps AND the loss on the train is:1.594948410987854\n",
      "0.9354\n",
      "the 255500 setps AND the loss on the train is:1.4851821660995483\n",
      "0.9356\n",
      "the 256000 setps AND the loss on the train is:1.5152345895767212\n",
      "0.9302\n",
      "the 256500 setps AND the loss on the train is:1.459587812423706\n",
      "0.9391\n",
      "the 257000 setps AND the loss on the train is:1.5058085918426514\n",
      "0.9349\n",
      "the 257500 setps AND the loss on the train is:1.6324764490127563\n",
      "0.9345\n",
      "the 258000 setps AND the loss on the train is:1.5083621740341187\n",
      "0.9371\n",
      "the 258500 setps AND the loss on the train is:1.5126888751983643\n",
      "0.9343\n",
      "the 259000 setps AND the loss on the train is:1.6150968074798584\n",
      "0.9368\n",
      "the 259500 setps AND the loss on the train is:1.505542278289795\n",
      "0.9315\n",
      "the 260000 setps AND the loss on the train is:1.5391777753829956\n",
      "0.9303\n",
      "the 260500 setps AND the loss on the train is:1.5969328880310059\n",
      "0.9349\n",
      "the 261000 setps AND the loss on the train is:1.478749394416809\n",
      "0.9373\n",
      "the 261500 setps AND the loss on the train is:1.5314419269561768\n",
      "0.9356\n",
      "the 262000 setps AND the loss on the train is:1.4894659519195557\n",
      "0.9312\n",
      "the 262500 setps AND the loss on the train is:1.4888529777526855\n",
      "0.9382\n",
      "the 263000 setps AND the loss on the train is:1.4990506172180176\n",
      "0.938\n",
      "the 263500 setps AND the loss on the train is:1.526910662651062\n",
      "0.9353\n",
      "the 264000 setps AND the loss on the train is:1.4913996458053589\n",
      "0.9357\n",
      "the 264500 setps AND the loss on the train is:1.5326207876205444\n",
      "0.9348\n",
      "the 265000 setps AND the loss on the train is:1.5228774547576904\n",
      "0.9304\n",
      "the 265500 setps AND the loss on the train is:1.4860860109329224\n",
      "0.9378\n",
      "the 266000 setps AND the loss on the train is:1.5207412242889404\n",
      "0.9365\n",
      "the 266500 setps AND the loss on the train is:1.6291029453277588\n",
      "0.9366\n",
      "the 267000 setps AND the loss on the train is:1.5261237621307373\n",
      "0.9367\n",
      "the 267500 setps AND the loss on the train is:1.5526622533798218\n",
      "0.9375\n",
      "the 268000 setps AND the loss on the train is:1.5549784898757935\n",
      "0.9357\n",
      "the 268500 setps AND the loss on the train is:1.568621039390564\n",
      "0.9361\n",
      "the 269000 setps AND the loss on the train is:1.5310016870498657\n",
      "0.9319\n",
      "the 269500 setps AND the loss on the train is:1.6454699039459229\n",
      "0.9352\n",
      "the 270000 setps AND the loss on the train is:1.5068244934082031\n",
      "0.932\n",
      "the 270500 setps AND the loss on the train is:1.4787089824676514\n",
      "0.9315\n",
      "the 271000 setps AND the loss on the train is:1.5660139322280884\n",
      "0.9295\n",
      "the 271500 setps AND the loss on the train is:1.4907135963439941\n",
      "0.936\n",
      "the 272000 setps AND the loss on the train is:1.5800460577011108\n",
      "0.9373\n",
      "the 272500 setps AND the loss on the train is:1.5633959770202637\n",
      "0.9345\n",
      "the 273000 setps AND the loss on the train is:1.5257902145385742\n",
      "0.9372\n",
      "the 273500 setps AND the loss on the train is:1.51290762424469\n",
      "0.9375\n",
      "the 274000 setps AND the loss on the train is:1.428754448890686\n",
      "0.9337\n",
      "the 274500 setps AND the loss on the train is:1.4740991592407227\n",
      "0.9367\n",
      "the 275000 setps AND the loss on the train is:1.459606409072876\n",
      "0.9372\n",
      "the 275500 setps AND the loss on the train is:1.5332108736038208\n",
      "0.9385\n",
      "the 276000 setps AND the loss on the train is:1.5151431560516357\n",
      "0.9352\n",
      "the 276500 setps AND the loss on the train is:1.440152645111084\n",
      "0.9364\n",
      "the 277000 setps AND the loss on the train is:1.5314445495605469\n",
      "0.9357\n",
      "the 277500 setps AND the loss on the train is:1.4927600622177124\n",
      "0.9381\n",
      "the 278000 setps AND the loss on the train is:1.4986234903335571\n",
      "0.9337\n",
      "the 278500 setps AND the loss on the train is:1.4295510053634644\n",
      "0.9359\n",
      "the 279000 setps AND the loss on the train is:1.5091931819915771\n",
      "0.9382\n",
      "the 279500 setps AND the loss on the train is:1.529781699180603\n",
      "0.935\n",
      "the 280000 setps AND the loss on the train is:1.4997695684432983\n",
      "0.9358\n",
      "the 280500 setps AND the loss on the train is:1.4723339080810547\n",
      "0.937\n",
      "the 281000 setps AND the loss on the train is:1.5213004350662231\n",
      "0.9351\n",
      "the 281500 setps AND the loss on the train is:1.5275561809539795\n",
      "0.9352\n",
      "the 282000 setps AND the loss on the train is:1.4846491813659668\n",
      "0.9362\n",
      "the 282500 setps AND the loss on the train is:1.4878662824630737\n",
      "0.9337\n",
      "the 283000 setps AND the loss on the train is:1.459112524986267\n",
      "0.9324\n",
      "the 283500 setps AND the loss on the train is:1.540549397468567\n",
      "0.9353\n",
      "the 284000 setps AND the loss on the train is:1.5365923643112183\n",
      "0.9314\n",
      "the 284500 setps AND the loss on the train is:1.4471778869628906\n",
      "0.9399\n",
      "the 285000 setps AND the loss on the train is:1.492772102355957\n",
      "0.9343\n",
      "the 285500 setps AND the loss on the train is:1.4947509765625\n",
      "0.9354\n",
      "the 286000 setps AND the loss on the train is:1.6027709245681763\n",
      "0.9352\n",
      "the 286500 setps AND the loss on the train is:1.5770907402038574\n",
      "0.9369\n",
      "the 287000 setps AND the loss on the train is:1.4754027128219604\n",
      "0.9383\n",
      "the 287500 setps AND the loss on the train is:1.4608546495437622\n",
      "0.9377\n",
      "the 288000 setps AND the loss on the train is:1.4712047576904297\n",
      "0.9396\n",
      "the 288500 setps AND the loss on the train is:1.4588215351104736\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9347\n",
      "the 289000 setps AND the loss on the train is:1.474449872970581\n",
      "0.9357\n",
      "the 289500 setps AND the loss on the train is:1.5031919479370117\n",
      "0.9376\n",
      "the 290000 setps AND the loss on the train is:1.5850825309753418\n",
      "0.9385\n",
      "the 290500 setps AND the loss on the train is:1.4946773052215576\n",
      "0.9371\n",
      "the 291000 setps AND the loss on the train is:1.545729160308838\n",
      "0.9373\n",
      "the 291500 setps AND the loss on the train is:1.5309100151062012\n",
      "0.9291\n",
      "the 292000 setps AND the loss on the train is:1.5009799003601074\n",
      "0.9332\n",
      "the 292500 setps AND the loss on the train is:1.5386773347854614\n",
      "0.9377\n",
      "the 293000 setps AND the loss on the train is:1.4364254474639893\n",
      "0.9334\n",
      "the 293500 setps AND the loss on the train is:1.4289461374282837\n",
      "0.936\n",
      "the 294000 setps AND the loss on the train is:1.4799529314041138\n",
      "0.9308\n",
      "the 294500 setps AND the loss on the train is:1.524799108505249\n",
      "0.9391\n",
      "the 295000 setps AND the loss on the train is:1.5053945779800415\n",
      "0.9331\n",
      "the 295500 setps AND the loss on the train is:1.5173205137252808\n",
      "0.937\n",
      "the 296000 setps AND the loss on the train is:1.558821678161621\n",
      "0.9351\n",
      "the 296500 setps AND the loss on the train is:1.4590835571289062\n",
      "0.9321\n",
      "the 297000 setps AND the loss on the train is:1.474483609199524\n",
      "0.9355\n",
      "the 297500 setps AND the loss on the train is:1.522857904434204\n",
      "0.9339\n",
      "the 298000 setps AND the loss on the train is:1.4567912817001343\n",
      "0.9348\n",
      "the 298500 setps AND the loss on the train is:1.5044655799865723\n",
      "0.9324\n",
      "the 299000 setps AND the loss on the train is:1.564339518547058\n",
      "0.9359\n",
      "the 299500 setps AND the loss on the train is:1.493895173072815\n",
      "0.9343\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,512],0.01)\n",
    "b1=tf.Variable(tf.random_normal([512]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.tanh(logits1)\n",
    "\n",
    "w2=get_weight([512,256],0.01)\n",
    "b2=tf.Variable(tf.random_normal([256]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.tanh(logits2)\n",
    "\n",
    "w3=get_weight([256,10],0.01)\n",
    "b3=tf.Variable(tf.random_normal([10]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "\n",
    "global_step=tf.Variable(0,trainable=False)\n",
    "learning_rate=tf.train.exponential_decay(0.1,global_step,500,0.99)\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits3))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits3,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(300000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4.5 与3.4.4相比，只将tanh激活转变为relu激活"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0 setps AND the loss on the train is:76038.4609375\n",
      "0.1921\n",
      "the 500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 1000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 1500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 2000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 2500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 3000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 3500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 4000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 4500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 5000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 5500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 6000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 6500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 7000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 7500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 8000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 8500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 9000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 9500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 10000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 10500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 11000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 11500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 12000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 12500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 13000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 13500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 14000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 14500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 15000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 15500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 16000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 16500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 17000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 17500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 18000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 18500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 19000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 19500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 20000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 20500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 21000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 21500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 22000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 22500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 23000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 23500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 24000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 24500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 25000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 25500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 26000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 26500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 27000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 27500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 28000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 28500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 29000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 29500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 30000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 30500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 31000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 31500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 32000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 32500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 33000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 33500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 34000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 34500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 35000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 35500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 36000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 36500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 37000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 37500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 38000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 38500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 39000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 39500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 40000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 40500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 41000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 41500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 42000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 42500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 43000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 43500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 44000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 44500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 45000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 45500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 46000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 46500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 47000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 47500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 48000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 48500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 49000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 49500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 50000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 50500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 51000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 51500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 52000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 52500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 53000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 53500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 54000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 54500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 55000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 55500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 56000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 56500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 57000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 57500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 58000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 58500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 59000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 59500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 60000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 60500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 61000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 61500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 62000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 62500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 63000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 63500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 64000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 64500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 65000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 65500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 66000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 66500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 67000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 67500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 68000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 68500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 69000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 69500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 70000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 70500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 71000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 71500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 72000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 72500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 73000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 73500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 74000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 74500 setps AND the loss on the train is:nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.098\n",
      "the 75000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 75500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 76000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 76500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 77000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 77500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 78000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 78500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 79000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 79500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 80000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 80500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 81000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 81500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 82000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 82500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 83000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 83500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 84000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 84500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 85000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 85500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 86000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 86500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 87000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 87500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 88000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 88500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 89000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 89500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 90000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 90500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 91000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 91500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 92000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 92500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 93000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 93500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 94000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 94500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 95000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 95500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 96000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 96500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 97000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 97500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 98000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 98500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 99000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 99500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 100000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 100500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 101000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 101500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 102000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 102500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 103000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 103500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 104000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 104500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 105000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 105500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 106000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 106500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 107000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 107500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 108000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 108500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 109000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 109500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 110000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 110500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 111000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 111500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 112000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 112500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 113000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 113500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 114000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 114500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 115000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 115500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 116000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 116500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 117000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 117500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 118000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 118500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 119000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 119500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 120000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 120500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 121000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 121500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 122000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 122500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 123000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 123500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 124000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 124500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 125000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 125500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 126000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 126500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 127000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 127500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 128000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 128500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 129000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 129500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 130000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 130500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 131000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 131500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 132000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 132500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 133000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 133500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 134000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 134500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 135000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 135500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 136000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 136500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 137000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 137500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 138000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 138500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 139000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 139500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 140000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 140500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 141000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 141500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 142000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 142500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 143000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 143500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 144000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 144500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 145000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 145500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 146000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 146500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 147000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 147500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 148000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 148500 setps AND the loss on the train is:nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.098\n",
      "the 149000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 149500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 150000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 150500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 151000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 151500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 152000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 152500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 153000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 153500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 154000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 154500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 155000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 155500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 156000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 156500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 157000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 157500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 158000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 158500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 159000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 159500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 160000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 160500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 161000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 161500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 162000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 162500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 163000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 163500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 164000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 164500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 165000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 165500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 166000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 166500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 167000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 167500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 168000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 168500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 169000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 169500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 170000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 170500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 171000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 171500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 172000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 172500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 173000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 173500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 174000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 174500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 175000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 175500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 176000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 176500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 177000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 177500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 178000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 178500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 179000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 179500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 180000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 180500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 181000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 181500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 182000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 182500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 183000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 183500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 184000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 184500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 185000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 185500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 186000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 186500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 187000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 187500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 188000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 188500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 189000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 189500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 190000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 190500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 191000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 191500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 192000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 192500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 193000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 193500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 194000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 194500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 195000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 195500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 196000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 196500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 197000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 197500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 198000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 198500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 199000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 199500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 200000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 200500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 201000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 201500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 202000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 202500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 203000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 203500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 204000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 204500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 205000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 205500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 206000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 206500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 207000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 207500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 208000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 208500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 209000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 209500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 210000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 210500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 211000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 211500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 212000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 212500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 213000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 213500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 214000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 214500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 215000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 215500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 216000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 216500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 217000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 217500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 218000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 218500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 219000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 219500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 220000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 220500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 221000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 221500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 222000 setps AND the loss on the train is:nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.098\n",
      "the 222500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 223000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 223500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 224000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 224500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 225000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 225500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 226000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 226500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 227000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 227500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 228000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 228500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 229000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 229500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 230000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 230500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 231000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 231500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 232000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 232500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 233000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 233500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 234000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 234500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 235000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 235500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 236000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 236500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 237000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 237500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 238000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 238500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 239000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 239500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 240000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 240500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 241000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 241500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 242000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 242500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 243000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 243500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 244000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 244500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 245000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 245500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 246000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 246500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 247000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 247500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 248000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 248500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 249000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 249500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 250000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 250500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 251000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 251500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 252000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 252500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 253000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 253500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 254000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 254500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 255000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 255500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 256000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 256500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 257000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 257500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 258000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 258500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 259000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 259500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 260000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 260500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 261000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 261500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 262000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 262500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 263000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 263500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 264000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 264500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 265000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 265500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 266000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 266500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 267000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 267500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 268000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 268500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 269000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 269500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 270000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 270500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 271000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 271500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 272000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 272500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 273000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 273500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 274000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 274500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 275000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 275500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 276000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 276500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 277000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 277500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 278000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 278500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 279000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 279500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 280000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 280500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 281000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 281500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 282000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 282500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 283000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 283500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 284000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 284500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 285000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 285500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 286000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 286500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 287000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 287500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 288000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 288500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 289000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 289500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 290000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 290500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 291000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 291500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 292000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 292500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 293000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 293500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 294000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 294500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 295000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 295500 setps AND the loss on the train is:nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.098\n",
      "the 296000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 296500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 297000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 297500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 298000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 298500 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 299000 setps AND the loss on the train is:nan\n",
      "0.098\n",
      "the 299500 setps AND the loss on the train is:nan\n",
      "0.098\n"
     ]
    }
   ],
   "source": [
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y_=tf.placeholder(tf.float32,[None,10])\n",
    "\n",
    "def get_weight(shape,lambd):\n",
    "    w=tf.Variable(tf.random_normal(shape),dtype=tf.float32)\n",
    "    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lambd)(w))\n",
    "    return w\n",
    "    \n",
    "    \n",
    "w1=get_weight([784,512],0.01)\n",
    "b1=tf.Variable(tf.random_normal([512]))\n",
    "logits1=tf.matmul(x,w1)+b1\n",
    "o1=tf.nn.relu(logits1)\n",
    "\n",
    "w2=get_weight([512,256],0.01)\n",
    "b2=tf.Variable(tf.random_normal([256]))\n",
    "logits2=tf.matmul(o1,w2)+b2\n",
    "o2=tf.nn.relu(logits2)\n",
    "\n",
    "w3=get_weight([256,10],0.01)\n",
    "b3=tf.Variable(tf.random_normal([10]))\n",
    "logits3=tf.matmul(o2,w3)+b3\n",
    "\n",
    "global_step=tf.Variable(0,trainable=False)\n",
    "learning_rate=tf.train.exponential_decay(0.1,global_step,500,0.99)\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=logits3))+tf.add_n(tf.get_collection('losses'))\n",
    "train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "correct_prediction=tf.equal(tf.argmax(logits3,1),tf.argmax(y_,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "\n",
    "sess=tf.Session()\n",
    "init_op=tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for i in range(300000):\n",
    "    batch_xs,batch_ys=mnist.train.next_batch(100)\n",
    "    sess.run(train_step,feed_dict={x:batch_xs,y_:batch_ys})\n",
    "    if i%500==0:\n",
    "        #感觉这样写有问题，为什么不能直接写sess.run(loss)就可以有输出呢\n",
    "        print('the {} setps AND the loss on the train is:{}'.format(i,sess.run(loss,feed_dict={x:batch_xs,y_:batch_ys})))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y_:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
